krz/gitbay

A CLI-first git forge. cli forge git self-hosted https://gitbay.org

Commit 51386c203f

51386c203f67e5ced14804b138d522e961c739d9

parent: 0b531fd6ad

Verified · cmc ci/build: success ci/test: failure ci/vuln: success

cmc <hello@cleberg.net> · 2026-09-02 02:55 UTC 

lfs: count it in stats, and gc --lfs for orphaned objects

admin stats walked repository directories only, so the LFS root was
missing from the disk total, and the Admin page recorded that gc never
collected an orphaned object.

stats carries lfs_bytes. gc --lfs scans every repository's objects for
LFS pointers, every object rather than the reachable ones since an
unreachable blob is still one gc has not removed, and deletes stored
objects no pointer names. Objects younger than a day are kept: git lfs
uploads before it pushes, so a fresh unreferenced object is a push in
progress, not an orphan.

Closes #79

Layout: unified · split

cmd/gitbayd/maint.go +48 −3
@@ -4,21 +4,23 @@ import (
4 "fmt" 4 "fmt"
5 "os" 5 "os"
6 "os/exec" 6 "os/exec"
7 "time"
7 8
8 "github.com/spf13/cobra" 9 "github.com/spf13/cobra"
9 10
10 "gitbay.org/gitbay/internal/config" 11 "gitbay.org/gitbay/internal/config"
11 "gitbay.org/gitbay/internal/control" 12 "gitbay.org/gitbay/internal/control"
12 "gitbay.org/gitbay/internal/gitutil" 13 "gitbay.org/gitbay/internal/gitutil"
14 "gitbay.org/gitbay/internal/lfs"
13 "gitbay.org/gitbay/internal/store" 15 "gitbay.org/gitbay/internal/store"
14) 16)
15 17
16func gcCmd() *cobra.Command { 18func gcCmd() *cobra.Command {
17 var repoPath string 19 var repoPath string
18 var aggressive bool 20 var aggressive, withLFS bool
19 cmd := &cobra.Command{ 21 cmd := &cobra.Command{
20 Use: "gc", 22 Use: "gc",
21 Short: "repack and prune repositories (git gc)", 23 Short: "repack and prune repositories (git gc); --lfs removes unreferenced LFS objects",
22 RunE: func(cmd *cobra.Command, args []string) error { 24 RunE: func(cmd *cobra.Command, args []string) error {
23 cfg, err := config.Load(configPath) 25 cfg, err := config.Load(configPath)
24 if err != nil { 26 if err != nil {
@@ -58,11 +60,15 @@ func gcCmd() *cobra.Command {
58 fmt.Printf("%s\t%s -> %s\n", r.Path(), human(b), human(a)) 60 fmt.Printf("%s\t%s -> %s\n", r.Path(), human(b), human(a))
59 } 61 }
60 fmt.Printf("total\t%s -> %s (freed %s)\n", human(before), human(after), human(before-after)) 62 fmt.Printf("total\t%s -> %s (freed %s)\n", human(before), human(after), human(before-after))
61 return nil 63 if !withLFS {
64 return nil
65 }
66 return gcLFS(cfg, st)
62 }, 67 },
63 } 68 }
64 cmd.Flags().StringVar(&repoPath, "repo", "", "one repository (owner/name) instead of all") 69 cmd.Flags().StringVar(&repoPath, "repo", "", "one repository (owner/name) instead of all")
65 cmd.Flags().BoolVar(&aggressive, "aggressive", false, "more thorough repack (slow; rarely needed)") 70 cmd.Flags().BoolVar(&aggressive, "aggressive", false, "more thorough repack (slow; rarely needed)")
71 cmd.Flags().BoolVar(&withLFS, "lfs", false, "also remove LFS objects no repository references (older than a day)")
66 return cmd 72 return cmd
67} 73}
68 74
@@ -78,3 +84,42 @@ func human(b int64) string {
78 return fmt.Sprintf("%d B", b) 84 return fmt.Sprintf("%d B", b)
79 } 85 }
80} 86}
87
88// lfsOrphanAge keeps an object uploaded ahead of the push that references
89// it: git lfs uploads first and pushes second.
90const lfsOrphanAge = 24 * time.Hour
91
92// gcLFS removes LFS objects no repository's pointers name. Every
93// repository is scanned, whatever --repo said: an object is shared across
94// repositories by content, so only the whole set says it is unreferenced.
95func gcLFS(cfg config.Config, st *store.Store) error {
96 repos, err := st.ListAllRepos()
97 if err != nil {
98 return err
99 }
100 referenced := map[string]bool{}
101 for _, r := range repos {
102 oids, err := gitutil.LFSPointerOIDs(control.RepoDir(cfg.Server.Root, r.OwnerName, r.Name))
103 if err != nil {
104 return fmt.Errorf("%s: scanning for LFS pointers: %w", r.Path(), err)
105 }
106 for _, o := range oids {
107 referenced[o] = true
108 }
109 }
110 blobs := lfs.LocalStore{Root: lfs.RootFor(cfg.LFS.Root, cfg.Server.Root)}
111 orphans, err := blobs.Orphans(referenced, lfsOrphanAge)
112 if err != nil {
113 return err
114 }
115 var freed int64
116 for _, o := range orphans {
117 if err := blobs.Delete(o.OID); err != nil {
118 fmt.Fprintf(os.Stderr, "lfs %s: %v\n", o.OID, err)
119 continue
120 }
121 freed += o.Size
122 }
123 fmt.Printf("lfs\t%d referenced, removed %d orphans (%s)\n", len(referenced), len(orphans), human(freed))
124 return nil
125}
e2e/reap_test.go +52
@@ -1,6 +1,8 @@
1package e2e 1package e2e
2 2
3import ( 3import (
4 "crypto/sha256"
5 "encoding/hex"
4 "encoding/json" 6 "encoding/json"
5 "fmt" 7 "fmt"
6 "os" 8 "os"
@@ -152,3 +154,53 @@ func TestHealthz(t *testing.T) {
152 t.Fatalf("healthz registered as a username: %s", errOut) 154 t.Fatalf("healthz registered as a username: %s", errOut)
153 } 155 }
154} 156}
157
158// gc --lfs removes objects no pointer names, keeps referenced ones, and
159// leaves anything young enough to be an upload ahead of its push.
160func TestGCLFSOrphans(t *testing.T) {
161 inst := startInstance(t)
162 aliceKey := inst.newKey(t, "alice")
163 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
164 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/big"); code != 0 {
165 t.Fatal("repo create failed")
166 }
167 put := func(content string, age time.Duration) string {
168 t.Helper()
169 sum := sha256.Sum256([]byte(content))
170 oid := hex.EncodeToString(sum[:])
171 path := filepath.Join(inst.root, "lfs", oid[:2], oid[2:4], oid)
172 os.MkdirAll(filepath.Dir(path), 0o755)
173 os.WriteFile(path, []byte(content), 0o644)
174 os.Chtimes(path, time.Now().Add(-age), time.Now().Add(-age))
175 return oid
176 }
177 kept := put("referenced payload", 48*time.Hour)
178 orphan := put("nobody points here", 48*time.Hour)
179 young := put("just uploaded", time.Minute)
180
181 work := t.TempDir()
182 env := inst.gitEnv(aliceKey)
183 mustGit(t, work, env, "clone", inst.sshURL("alice/big"), "w")
184 dir := filepath.Join(work, "w")
185 pointer := fmt.Sprintf("version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %d\n", kept, len("referenced payload"))
186 os.WriteFile(filepath.Join(dir, "data.bin"), []byte(pointer), 0o644)
187 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
188 mustGit(t, dir, env, "add", ".")
189 mustGit(t, dir, env, "commit", "-q", "-m", "pointer")
190 mustGit(t, dir, env, "push", "-q", "origin", "main")
191
192 if out := inst.admin(t, "admin", "stats", "--json"); !strings.Contains(out, `"lfs_bytes":`) || strings.Contains(out, `"lfs_bytes":0`) {
193 t.Fatalf("stats lfs bytes:\n%s", out)
194 }
195 out := inst.admin(t, "admin", "gc", "--lfs")
196 if !strings.Contains(out, "lfs\t1 referenced, removed 1 orphans") {
197 t.Fatalf("gc --lfs:\n%s", out)
198 }
199 exists := func(oid string) bool {
200 _, err := os.Stat(filepath.Join(inst.root, "lfs", oid[:2], oid[2:4], oid))
201 return err == nil
202 }
203 if !exists(kept) || exists(orphan) || !exists(young) {
204 t.Fatalf("after gc: kept=%v orphan=%v young=%v", exists(kept), exists(orphan), exists(young))
205 }
206}
internal/control/adminhost.go +4 −1
@@ -9,6 +9,7 @@ import (
9 "golang.org/x/crypto/ssh" 9 "golang.org/x/crypto/ssh"
10 10
11 "gitbay.org/gitbay/internal/gitutil" 11 "gitbay.org/gitbay/internal/gitutil"
12 "gitbay.org/gitbay/internal/lfs"
12 "gitbay.org/gitbay/internal/mail" 13 "gitbay.org/gitbay/internal/mail"
13 "gitbay.org/gitbay/internal/policy" 14 "gitbay.org/gitbay/internal/policy"
14 "gitbay.org/gitbay/internal/protocol" 15 "gitbay.org/gitbay/internal/protocol"
@@ -301,9 +302,11 @@ func runAdminStats(c *Ctx, args []string) int {
301 Counts store.Counts `json:"counts"` 302 Counts store.Counts `json:"counts"`
302 DBBytes int64 `json:"db_bytes"` 303 DBBytes int64 `json:"db_bytes"`
303 RepoBytes int64 `json:"repo_bytes"` 304 RepoBytes int64 `json:"repo_bytes"`
305 LFSBytes int64 `json:"lfs_bytes"`
304 Repos []repoDisk `json:"repos"` 306 Repos []repoDisk `json:"repos"`
305 } 307 }
306 d := out{Counts: counts, Repos: []repoDisk{}} 308 d := out{Counts: counts, Repos: []repoDisk{}}
309 d.LFSBytes = lfs.LocalStore{Root: lfs.RootFor(c.Cfg.LFS.Root, c.Cfg.Server.Root)}.Size()
307 for _, r := range repos { 310 for _, r := range repos {
308 b := gitutil.DirSize(RepoDir(c.Cfg.Server.Root, r.OwnerName, r.Name)) 311 b := gitutil.DirSize(RepoDir(c.Cfg.Server.Root, r.OwnerName, r.Name))
309 d.Repos = append(d.Repos, repoDisk{r.Path(), b}) 312 d.Repos = append(d.Repos, repoDisk{r.Path(), b})
@@ -316,7 +319,7 @@ func runAdminStats(c *Ctx, args []string) int {
316 fmt.Fprintf(w, "users %d · orgs %d · repos %d · issues %d (%d open) · MRs %d (%d open)\n", 319 fmt.Fprintf(w, "users %d · orgs %d · repos %d · issues %d (%d open) · MRs %d (%d open)\n",
317 counts.Users, counts.Orgs, counts.Repos, 320 counts.Users, counts.Orgs, counts.Repos,
318 counts.Issues, counts.OpenIssues, counts.MRs, counts.OpenMRs) 321 counts.Issues, counts.OpenIssues, counts.MRs, counts.OpenMRs)
319 fmt.Fprintf(w, "database %s · repositories %s\n\n", humanBytes(d.DBBytes), humanBytes(d.RepoBytes)) 322 fmt.Fprintf(w, "database %s · repositories %s · lfs %s\n\n", humanBytes(d.DBBytes), humanBytes(d.RepoBytes), humanBytes(d.LFSBytes))
320 for _, r := range d.Repos { 323 for _, r := range d.Repos {
321 fmt.Fprintf(w, "%s\t%s\n", r.Path, humanBytes(r.Bytes)) 324 fmt.Fprintf(w, "%s\t%s\n", r.Path, humanBytes(r.Bytes))
322 } 325 }
internal/gitutil/lfs.go added +77
@@ -0,0 +1,77 @@
1package gitutil
2
3import (
4 "bufio"
5 "bytes"
6 "os/exec"
7 "regexp"
8 "strconv"
9 "strings"
10)
11
12// lfsPointerMax bounds a pointer file; real ones are around 130 bytes.
13const lfsPointerMax = 1024
14
15var lfsOIDLine = regexp.MustCompile(`(?m)^oid sha256:([0-9a-f]{64})$`)
16
17// LFSPointerOIDs returns every LFS object id referenced by a pointer blob
18// anywhere in the repository: every object, not just the reachable ones,
19// since an unreachable blob is still an object gc has not removed.
20func LFSPointerOIDs(dir string) ([]string, error) {
21 list := exec.Command("git", "-C", dir, "cat-file", "--batch-all-objects",
22 "--batch-check=%(objecttype) %(objectsize) %(objectname)")
23 out, err := list.Output()
24 if err != nil {
25 return nil, err
26 }
27 var small []string
28 sc := bufio.NewScanner(bytes.NewReader(out))
29 for sc.Scan() {
30 f := strings.Fields(sc.Text())
31 if len(f) != 3 || f[0] != "blob" {
32 continue
33 }
34 if n, err := strconv.Atoi(f[1]); err == nil && n <= lfsPointerMax {
35 small = append(small, f[2])
36 }
37 }
38 if len(small) == 0 {
39 return nil, nil
40 }
41 cat := exec.Command("git", "-C", dir, "cat-file", "--batch")
42 cat.Stdin = strings.NewReader(strings.Join(small, "\n") + "\n")
43 out, err = cat.Output()
44 if err != nil {
45 return nil, err
46 }
47 // --batch output: "<sha> blob <size>\n<content>\n" per object.
48 var oids []string
49 rest := out
50 for len(rest) > 0 {
51 nl := bytes.IndexByte(rest, '\n')
52 if nl < 0 {
53 break
54 }
55 hdr := strings.Fields(string(rest[:nl]))
56 rest = rest[nl+1:]
57 if len(hdr) != 3 {
58 break
59 }
60 size, _ := strconv.Atoi(hdr[2])
61 if size > len(rest) {
62 break
63 }
64 body := rest[:size]
65 rest = rest[size:]
66 if len(rest) > 0 && rest[0] == '\n' {
67 rest = rest[1:]
68 }
69 if !bytes.HasPrefix(body, []byte("version https://git-lfs.github.com/spec/")) {
70 continue
71 }
72 if m := lfsOIDLine.FindSubmatch(body); m != nil {
73 oids = append(oids, string(m[1]))
74 }
75 }
76 return oids, nil
77}
internal/httpd/lfs.go +1 −6
@@ -5,7 +5,6 @@ import (
5 "fmt" 5 "fmt"
6 "io" 6 "io"
7 "net/http" 7 "net/http"
8 "path/filepath"
9 "strings" 8 "strings"
10 "time" 9 "time"
11 10
@@ -21,11 +20,7 @@ import (
21const lfsMediaType = "application/vnd.git-lfs+json" 20const lfsMediaType = "application/vnd.git-lfs+json"
22 21
23func (s *Server) lfsStore() lfs.BlobStore { 22func (s *Server) lfsStore() lfs.BlobStore {
24 root := s.cfg.LFS.Root 23 return lfs.LocalStore{Root: lfs.RootFor(s.cfg.LFS.Root, s.cfg.Server.Root)}
25 if root == "" {
26 root = filepath.Join(s.cfg.Server.Root, "lfs")
27 }
28 return lfs.LocalStore{Root: root}
29} 24}
30 25
31func (s *Server) lfsMaxObject() int64 { 26func (s *Server) lfsMaxObject() int64 {
internal/lfs/lfs.go +54
@@ -20,6 +20,7 @@ import (
20 "encoding/hex" 20 "encoding/hex"
21 "fmt" 21 "fmt"
22 "io" 22 "io"
23 "io/fs"
23 "os" 24 "os"
24 "path/filepath" 25 "path/filepath"
25 "regexp" 26 "regexp"
@@ -173,3 +174,56 @@ func NewSecret() string {
173 rand.Read(buf) 174 rand.Read(buf)
174 return hex.EncodeToString(buf) 175 return hex.EncodeToString(buf)
175} 176}
177
178// Orphans lists objects in the store that no repository references and
179// that are older than minAge: an object uploaded ahead of the push that
180// will reference it is not an orphan yet. referenced holds the object ids
181// every repository's pointers name.
182func (s LocalStore) Orphans(referenced map[string]bool, minAge time.Duration) ([]Orphan, error) {
183 cutoff := time.Now().Add(-minAge)
184 var out []Orphan
185 err := filepath.WalkDir(s.Root, func(path string, d fs.DirEntry, err error) error {
186 if err != nil || d.IsDir() {
187 return nil
188 }
189 oid := d.Name()
190 if !OIDPat.MatchString(oid) || referenced[oid] {
191 return nil
192 }
193 info, err := d.Info()
194 if err != nil || info.ModTime().After(cutoff) {
195 return nil
196 }
197 out = append(out, Orphan{OID: oid, Size: info.Size()})
198 return nil
199 })
200 return out, err
201}
202
203// Orphan is one unreferenced object.
204type Orphan struct {
205 OID string
206 Size int64
207}
208
209// Size sums every object in the store.
210func (s LocalStore) Size() int64 {
211 var total int64
212 filepath.WalkDir(s.Root, func(_ string, d fs.DirEntry, err error) error {
213 if err == nil && !d.IsDir() {
214 if fi, err := d.Info(); err == nil {
215 total += fi.Size()
216 }
217 }
218 return nil
219 })
220 return total
221}
222
223// RootFor is the store root a configuration implies.
224func RootFor(lfsRoot, serverRoot string) string {
225 if lfsRoot != "" {
226 return lfsRoot
227 }
228 return filepath.Join(serverRoot, "lfs")
229}