internal/lfs/lfs.go

  1// Package lfs implements Git LFS server storage and authorization.
  2//
  3// The protocol surface lives in httpd (batch API + basic transfers) and
  4// sshd (git-lfs-authenticate); this package owns the pieces both need:
  5// content-addressed blob storage behind a small interface, and the
  6// short-lived tokens that bridge SSH authentication to the HTTP endpoints.
  7//
  8// BlobStore is deliberately minimal so an S3-compatible backend is a
  9// drop-in: implement the four methods against a bucket and the batch and
 10// transfer handlers work unchanged (the server streams as a proxy).
 11// Handing clients presigned URLs instead is a later optimization to the
 12// batch handler, not a rewrite.
 13package lfs
 14
 15import (
 16	"crypto/hmac"
 17	"crypto/rand"
 18	"crypto/sha256"
 19	"encoding/base64"
 20	"encoding/hex"
 21	"fmt"
 22	"io"
 23	"io/fs"
 24	"os"
 25	"path/filepath"
 26	"regexp"
 27	"strconv"
 28	"strings"
 29	"time"
 30)
 31
 32// OIDPat is a lowercase sha256 hex digest — the only object name LFS uses.
 33var OIDPat = regexp.MustCompile(`^[a-f0-9]{64}$`)
 34
 35// BlobStore holds LFS objects by their sha256 content address.
 36type BlobStore interface {
 37	// Put stores the reader's content as oid, verifying both size and
 38	// digest; a mismatch stores nothing.
 39	Put(oid string, r io.Reader, size int64) error
 40	Get(oid string) (io.ReadCloser, int64, error)
 41	Exists(oid string) (int64, bool)
 42	Delete(oid string) error
 43}
 44
 45// LocalStore is the on-disk backend: <root>/<aa>/<bb>/<oid>, written via a
 46// temp file and renamed only after the digest checks out.
 47type LocalStore struct {
 48	Root string
 49}
 50
 51func (s LocalStore) path(oid string) string {
 52	return filepath.Join(s.Root, oid[:2], oid[2:4], oid)
 53}
 54
 55func (s LocalStore) Put(oid string, r io.Reader, size int64) error {
 56	if !OIDPat.MatchString(oid) {
 57		return fmt.Errorf("bad oid %q", oid)
 58	}
 59	dir := filepath.Dir(s.path(oid))
 60	if err := os.MkdirAll(dir, 0o755); err != nil {
 61		return err
 62	}
 63	tmp, err := os.CreateTemp(dir, ".upload-*")
 64	if err != nil {
 65		return err
 66	}
 67	defer func() {
 68		tmp.Close()
 69		os.Remove(tmp.Name())
 70	}()
 71	h := sha256.New()
 72	n, err := io.Copy(io.MultiWriter(tmp, h), io.LimitReader(r, size+1))
 73	if err != nil {
 74		return err
 75	}
 76	if n != size {
 77		return fmt.Errorf("size mismatch: got %d bytes, expected %d", n, size)
 78	}
 79	if sum := hex.EncodeToString(h.Sum(nil)); sum != oid {
 80		return fmt.Errorf("content digest %s does not match oid", sum[:12])
 81	}
 82	if err := tmp.Close(); err != nil {
 83		return err
 84	}
 85	return os.Rename(tmp.Name(), s.path(oid))
 86}
 87
 88func (s LocalStore) Get(oid string) (io.ReadCloser, int64, error) {
 89	if !OIDPat.MatchString(oid) {
 90		return nil, 0, fmt.Errorf("bad oid %q", oid)
 91	}
 92	f, err := os.Open(s.path(oid))
 93	if err != nil {
 94		return nil, 0, err
 95	}
 96	fi, err := f.Stat()
 97	if err != nil {
 98		f.Close()
 99		return nil, 0, err
100	}
101	return f, fi.Size(), nil
102}
103
104func (s LocalStore) Exists(oid string) (int64, bool) {
105	if !OIDPat.MatchString(oid) {
106		return 0, false
107	}
108	fi, err := os.Stat(s.path(oid))
109	if err != nil {
110		return 0, false
111	}
112	return fi.Size(), true
113}
114
115func (s LocalStore) Delete(oid string) error {
116	if !OIDPat.MatchString(oid) {
117		return fmt.Errorf("bad oid %q", oid)
118	}
119	return os.Remove(s.path(oid))
120}
121
122// Tokens bridge SSH authentication to the HTTP endpoints: stateless,
123// HMAC-signed, scoped to one repo and one operation, short-lived. The
124// secret persists in the settings table so tokens survive restarts.
125
126const TokenTTL = time.Hour
127
128// Sign mints a token for op ("download" or "upload") on repoID.
129func Sign(secret []byte, repoID int64, op string, now time.Time) string {
130	payload := fmt.Sprintf("%d:%s:%d", repoID, op, now.Add(TokenTTL).Unix())
131	mac := hmac.New(sha256.New, secret)
132	mac.Write([]byte(payload))
133	return base64.RawURLEncoding.EncodeToString([]byte(payload)) + "." +
134		base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
135}
136
137// Verify checks a token and returns the repo and operation it authorizes.
138func Verify(secret []byte, token string, now time.Time) (repoID int64, op string, ok bool) {
139	payloadB64, macB64, found := strings.Cut(token, ".")
140	if !found {
141		return 0, "", false
142	}
143	payload, err := base64.RawURLEncoding.DecodeString(payloadB64)
144	if err != nil {
145		return 0, "", false
146	}
147	gotMAC, err := base64.RawURLEncoding.DecodeString(macB64)
148	if err != nil {
149		return 0, "", false
150	}
151	mac := hmac.New(sha256.New, secret)
152	mac.Write(payload)
153	if !hmac.Equal(mac.Sum(nil), gotMAC) {
154		return 0, "", false
155	}
156	parts := strings.Split(string(payload), ":")
157	if len(parts) != 3 {
158		return 0, "", false
159	}
160	id, err1 := strconv.ParseInt(parts[0], 10, 64)
161	exp, err2 := strconv.ParseInt(parts[2], 10, 64)
162	if err1 != nil || err2 != nil || now.Unix() > exp {
163		return 0, "", false
164	}
165	if parts[1] != "download" && parts[1] != "upload" {
166		return 0, "", false
167	}
168	return id, parts[1], true
169}
170
171// NewSecret returns 32 random bytes, hex-encoded for the settings table.
172func NewSecret() string {
173	buf := make([]byte, 32)
174	rand.Read(buf)
175	return hex.EncodeToString(buf)
176}
177
178// Orphans lists objects in the store that no repository references and
179// that are older than minAge: an object uploaded ahead of the push that
180// will reference it is not an orphan yet. referenced holds the object ids
181// every repository's pointers name.
182func (s LocalStore) Orphans(referenced map[string]bool, minAge time.Duration) ([]Orphan, error) {
183	cutoff := time.Now().Add(-minAge)
184	var out []Orphan
185	err := filepath.WalkDir(s.Root, func(path string, d fs.DirEntry, err error) error {
186		if err != nil || d.IsDir() {
187			return nil
188		}
189		oid := d.Name()
190		if !OIDPat.MatchString(oid) || referenced[oid] {
191			return nil
192		}
193		info, err := d.Info()
194		if err != nil || info.ModTime().After(cutoff) {
195			return nil
196		}
197		out = append(out, Orphan{OID: oid, Size: info.Size()})
198		return nil
199	})
200	return out, err
201}
202
203// Orphan is one unreferenced object.
204type Orphan struct {
205	OID  string
206	Size int64
207}
208
209// Size sums every object in the store.
210func (s LocalStore) Size() int64 {
211	var total int64
212	filepath.WalkDir(s.Root, func(_ string, d fs.DirEntry, err error) error {
213		if err == nil && !d.IsDir() {
214			if fi, err := d.Info(); err == nil {
215				total += fi.Size()
216			}
217		}
218		return nil
219	})
220	return total
221}
222
223// RootFor is the store root a configuration implies.
224func RootFor(lfsRoot, serverRoot string) string {
225	if lfsRoot != "" {
226		return lfsRoot
227	}
228	return filepath.Join(serverRoot, "lfs")
229}