!163 · krz/gitbay

cmd/gitbayd/maint.go +48 −3

		@@ -4,21 +4,23 @@ import (
4	4	"fmt"
5	5	"os"
6	6	"os/exec"
	7	"time"
7	8
8	9	"github.com/spf13/cobra"
9	10
10	11	"gitbay.org/gitbay/internal/config"
11	12	"gitbay.org/gitbay/internal/control"
12	13	"gitbay.org/gitbay/internal/gitutil"
	14	"gitbay.org/gitbay/internal/lfs"
13	15	"gitbay.org/gitbay/internal/store"
14	16	)
15	17
16	18	func gcCmd() *cobra.Command {
17	19	var repoPath string
18		var aggressive bool
	20	var aggressive, withLFS bool
19	21	cmd := &cobra.Command{
20	22	Use: "gc",
21		Short: "repack and prune repositories (git gc)",
	23	Short: "repack and prune repositories (git gc); --lfs removes unreferenced LFS objects",
22	24	RunE: func(cmd *cobra.Command, args []string) error {
23	25	cfg, err := config.Load(configPath)
24	26	if err != nil {
		@@ -58,11 +60,15 @@ func gcCmd() *cobra.Command {
58	60	fmt.Printf("%s\t%s -> %s\n", r.Path(), human(b), human(a))
59	61	}
60	62	fmt.Printf("total\t%s -> %s (freed %s)\n", human(before), human(after), human(before-after))
61		return nil
	63	if !withLFS {
	64	return nil
	65	}
	66	return gcLFS(cfg, st)
62	67	},
63	68	}
64	69	cmd.Flags().StringVar(&repoPath, "repo", "", "one repository (owner/name) instead of all")
65	70	cmd.Flags().BoolVar(&aggressive, "aggressive", false, "more thorough repack (slow; rarely needed)")
	71	cmd.Flags().BoolVar(&withLFS, "lfs", false, "also remove LFS objects no repository references (older than a day)")
66	72	return cmd
67	73	}
68	74
		@@ -78,3 +84,42 @@ func human(b int64) string {
78	84	return fmt.Sprintf("%d B", b)
79	85	}
80	86	}
	87
	88	// lfsOrphanAge keeps an object uploaded ahead of the push that references
	89	// it: git lfs uploads first and pushes second.
	90	const lfsOrphanAge = 24 * time.Hour
	91
	92	// gcLFS removes LFS objects no repository's pointers name. Every
	93	// repository is scanned, whatever --repo said: an object is shared across
	94	// repositories by content, so only the whole set says it is unreferenced.
	95	func gcLFS(cfg config.Config, st *store.Store) error {
	96	repos, err := st.ListAllRepos()
	97	if err != nil {
	98	return err
	99	}
	100	referenced := map[string]bool{}
	101	for _, r := range repos {
	102	oids, err := gitutil.LFSPointerOIDs(control.RepoDir(cfg.Server.Root, r.OwnerName, r.Name))
	103	if err != nil {
	104	return fmt.Errorf("%s: scanning for LFS pointers: %w", r.Path(), err)
	105	}
	106	for _, o := range oids {
	107	referenced[o] = true
	108	}
	109	}
	110	blobs := lfs.LocalStore{Root: lfs.RootFor(cfg.LFS.Root, cfg.Server.Root)}
	111	orphans, err := blobs.Orphans(referenced, lfsOrphanAge)
	112	if err != nil {
	113	return err
	114	}
	115	var freed int64
	116	for _, o := range orphans {
	117	if err := blobs.Delete(o.OID); err != nil {
	118	fmt.Fprintf(os.Stderr, "lfs %s: %v\n", o.OID, err)
	119	continue
	120	}
	121	freed += o.Size
	122	}
	123	fmt.Printf("lfs\t%d referenced, removed %d orphans (%s)\n", len(referenced), len(orphans), human(freed))
	124	return nil
	125	}

e2e/reap_test.go +52

		@@ -1,6 +1,8 @@
1	1	package e2e
2	2
3	3	import (
	4	"crypto/sha256"
	5	"encoding/hex"
4	6	"encoding/json"
5	7	"fmt"
6	8	"os"
		@@ -152,3 +154,53 @@ func TestHealthz(t *testing.T) {
152	154	t.Fatalf("healthz registered as a username: %s", errOut)
153	155	}
154	156	}
	157
	158	// gc --lfs removes objects no pointer names, keeps referenced ones, and
	159	// leaves anything young enough to be an upload ahead of its push.
	160	func TestGCLFSOrphans(t *testing.T) {
	161	inst := startInstance(t)
	162	aliceKey := inst.newKey(t, "alice")
	163	inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
	164	if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/big"); code != 0 {
	165	t.Fatal("repo create failed")
	166	}
	167	put := func(content string, age time.Duration) string {
	168	t.Helper()
	169	sum := sha256.Sum256([]byte(content))
	170	oid := hex.EncodeToString(sum[:])
	171	path := filepath.Join(inst.root, "lfs", oid[:2], oid[2:4], oid)
	172	os.MkdirAll(filepath.Dir(path), 0o755)
	173	os.WriteFile(path, []byte(content), 0o644)
	174	os.Chtimes(path, time.Now().Add(-age), time.Now().Add(-age))
	175	return oid
	176	}
	177	kept := put("referenced payload", 48*time.Hour)
	178	orphan := put("nobody points here", 48*time.Hour)
	179	young := put("just uploaded", time.Minute)
	180
	181	work := t.TempDir()
	182	env := inst.gitEnv(aliceKey)
	183	mustGit(t, work, env, "clone", inst.sshURL("alice/big"), "w")
	184	dir := filepath.Join(work, "w")
	185	pointer := fmt.Sprintf("version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %d\n", kept, len("referenced payload"))
	186	os.WriteFile(filepath.Join(dir, "data.bin"), []byte(pointer), 0o644)
	187	mustGit(t, dir, env, "checkout", "-q", "-b", "main")
	188	mustGit(t, dir, env, "add", ".")
	189	mustGit(t, dir, env, "commit", "-q", "-m", "pointer")
	190	mustGit(t, dir, env, "push", "-q", "origin", "main")
	191
	192	if out := inst.admin(t, "admin", "stats", "--json"); !strings.Contains(out, `"lfs_bytes":`) \|\| strings.Contains(out, `"lfs_bytes":0`) {
	193	t.Fatalf("stats lfs bytes:\n%s", out)
	194	}
	195	out := inst.admin(t, "admin", "gc", "--lfs")
	196	if !strings.Contains(out, "lfs\t1 referenced, removed 1 orphans") {
	197	t.Fatalf("gc --lfs:\n%s", out)
	198	}
	199	exists := func(oid string) bool {
	200	_, err := os.Stat(filepath.Join(inst.root, "lfs", oid[:2], oid[2:4], oid))
	201	return err == nil
	202	}
	203	if !exists(kept) \|\| exists(orphan) \|\| !exists(young) {
	204	t.Fatalf("after gc: kept=%v orphan=%v young=%v", exists(kept), exists(orphan), exists(young))
	205	}
	206	}

internal/control/adminhost.go +4 −1

		@@ -9,6 +9,7 @@ import (
9	9	"golang.org/x/crypto/ssh"
10	10
11	11	"gitbay.org/gitbay/internal/gitutil"
	12	"gitbay.org/gitbay/internal/lfs"
12	13	"gitbay.org/gitbay/internal/mail"
13	14	"gitbay.org/gitbay/internal/policy"
14	15	"gitbay.org/gitbay/internal/protocol"
		@@ -301,9 +302,11 @@ func runAdminStats(c *Ctx, args []string) int {
301	302	Counts store.Counts `json:"counts"`
302	303	DBBytes int64 `json:"db_bytes"`
303	304	RepoBytes int64 `json:"repo_bytes"`
	305	LFSBytes int64 `json:"lfs_bytes"`
304	306	Repos []repoDisk `json:"repos"`
305	307	}
306	308	d := out{Counts: counts, Repos: []repoDisk{}}
	309	d.LFSBytes = lfs.LocalStore{Root: lfs.RootFor(c.Cfg.LFS.Root, c.Cfg.Server.Root)}.Size()
307	310	for _, r := range repos {
308	311	b := gitutil.DirSize(RepoDir(c.Cfg.Server.Root, r.OwnerName, r.Name))
309	312	d.Repos = append(d.Repos, repoDisk{r.Path(), b})
		@@ -316,7 +319,7 @@ func runAdminStats(c *Ctx, args []string) int {
316	319	fmt.Fprintf(w, "users %d · orgs %d · repos %d · issues %d (%d open) · MRs %d (%d open)\n",
317	320	counts.Users, counts.Orgs, counts.Repos,
318	321	counts.Issues, counts.OpenIssues, counts.MRs, counts.OpenMRs)
319		fmt.Fprintf(w, "database %s · repositories %s\n\n", humanBytes(d.DBBytes), humanBytes(d.RepoBytes))
	322	fmt.Fprintf(w, "database %s · repositories %s · lfs %s\n\n", humanBytes(d.DBBytes), humanBytes(d.RepoBytes), humanBytes(d.LFSBytes))
320	323	for _, r := range d.Repos {
321	324	fmt.Fprintf(w, "%s\t%s\n", r.Path, humanBytes(r.Bytes))
322	325	}

internal/gitutil/lfs.go added +77

		@@ -0,0 +1,77 @@
	1	package gitutil
	2
	3	import (
	4	"bufio"
	5	"bytes"
	6	"os/exec"
	7	"regexp"
	8	"strconv"
	9	"strings"
	10	)
	11
	12	// lfsPointerMax bounds a pointer file; real ones are around 130 bytes.
	13	const lfsPointerMax = 1024
	14
	15	var lfsOIDLine = regexp.MustCompile(`(?m)^oid sha256:([0-9a-f]{64})$`)
	16
	17	// LFSPointerOIDs returns every LFS object id referenced by a pointer blob
	18	// anywhere in the repository: every object, not just the reachable ones,
	19	// since an unreachable blob is still an object gc has not removed.
	20	func LFSPointerOIDs(dir string) ([]string, error) {
	21	list := exec.Command("git", "-C", dir, "cat-file", "--batch-all-objects",
	22	"--batch-check=%(objecttype) %(objectsize) %(objectname)")
	23	out, err := list.Output()
	24	if err != nil {
	25	return nil, err
	26	}
	27	var small []string
	28	sc := bufio.NewScanner(bytes.NewReader(out))
	29	for sc.Scan() {
	30	f := strings.Fields(sc.Text())
	31	if len(f) != 3 \|\| f[0] != "blob" {
	32	continue
	33	}
	34	if n, err := strconv.Atoi(f[1]); err == nil && n <= lfsPointerMax {
	35	small = append(small, f[2])
	36	}
	37	}
	38	if len(small) == 0 {
	39	return nil, nil
	40	}
	41	cat := exec.Command("git", "-C", dir, "cat-file", "--batch")
	42	cat.Stdin = strings.NewReader(strings.Join(small, "\n") + "\n")
	43	out, err = cat.Output()
	44	if err != nil {
	45	return nil, err
	46	}
	47	// --batch output: "<sha> blob <size>\n<content>\n" per object.
	48	var oids []string
	49	rest := out
	50	for len(rest) > 0 {
	51	nl := bytes.IndexByte(rest, '\n')
	52	if nl < 0 {
	53	break
	54	}
	55	hdr := strings.Fields(string(rest[:nl]))
	56	rest = rest[nl+1:]
	57	if len(hdr) != 3 {
	58	break
	59	}
	60	size, _ := strconv.Atoi(hdr[2])
	61	if size > len(rest) {
	62	break
	63	}
	64	body := rest[:size]
	65	rest = rest[size:]
	66	if len(rest) > 0 && rest[0] == '\n' {
	67	rest = rest[1:]
	68	}
	69	if !bytes.HasPrefix(body, []byte("version https://git-lfs.github.com/spec/")) {
	70	continue
	71	}
	72	if m := lfsOIDLine.FindSubmatch(body); m != nil {
	73	oids = append(oids, string(m[1]))
	74	}
	75	}
	76	return oids, nil
	77	}

internal/httpd/lfs.go +1 −6

		@@ -5,7 +5,6 @@ import (
5	5	"fmt"
6	6	"io"
7	7	"net/http"
8		"path/filepath"
9	8	"strings"
10	9	"time"
11	10
		@@ -21,11 +20,7 @@ import (
21	20	const lfsMediaType = "application/vnd.git-lfs+json"
22	21
23	22	func (s *Server) lfsStore() lfs.BlobStore {
24		root := s.cfg.LFS.Root
25		if root == "" {
26		root = filepath.Join(s.cfg.Server.Root, "lfs")
27		}
28		return lfs.LocalStore{Root: root}
	23	return lfs.LocalStore{Root: lfs.RootFor(s.cfg.LFS.Root, s.cfg.Server.Root)}
29	24	}
30	25
31	26	func (s *Server) lfsMaxObject() int64 {

internal/lfs/lfs.go +54

		@@ -20,6 +20,7 @@ import (
20	20	"encoding/hex"
21	21	"fmt"
22	22	"io"
	23	"io/fs"
23	24	"os"
24	25	"path/filepath"
25	26	"regexp"
		@@ -173,3 +174,56 @@ func NewSecret() string {
173	174	rand.Read(buf)
174	175	return hex.EncodeToString(buf)
175	176	}
	177
	178	// Orphans lists objects in the store that no repository references and
	179	// that are older than minAge: an object uploaded ahead of the push that
	180	// will reference it is not an orphan yet. referenced holds the object ids
	181	// every repository's pointers name.
	182	func (s LocalStore) Orphans(referenced map[string]bool, minAge time.Duration) ([]Orphan, error) {
	183	cutoff := time.Now().Add(-minAge)
	184	var out []Orphan
	185	err := filepath.WalkDir(s.Root, func(path string, d fs.DirEntry, err error) error {
	186	if err != nil \|\| d.IsDir() {
	187	return nil
	188	}
	189	oid := d.Name()
	190	if !OIDPat.MatchString(oid) \|\| referenced[oid] {
	191	return nil
	192	}
	193	info, err := d.Info()
	194	if err != nil \|\| info.ModTime().After(cutoff) {
	195	return nil
	196	}
	197	out = append(out, Orphan{OID: oid, Size: info.Size()})
	198	return nil
	199	})
	200	return out, err
	201	}
	202
	203	// Orphan is one unreferenced object.
	204	type Orphan struct {
	205	OID string
	206	Size int64
	207	}
	208
	209	// Size sums every object in the store.
	210	func (s LocalStore) Size() int64 {
	211	var total int64
	212	filepath.WalkDir(s.Root, func(_ string, d fs.DirEntry, err error) error {
	213	if err == nil && !d.IsDir() {
	214	if fi, err := d.Info(); err == nil {
	215	total += fi.Size()
	216	}
	217	}
	218	return nil
	219	})
	220	return total
	221	}
	222
	223	// RootFor is the store root a configuration implies.
	224	func RootFor(lfsRoot, serverRoot string) string {
	225	if lfsRoot != "" {
	226	return lfsRoot
	227	}
	228	return filepath.Join(serverRoot, "lfs")
	229	}

lfs: count it in stats, and gc --lfs for orphaned objects !163