internal/lfs/lfs.go
260 lines · 7627 bytes
1// Package lfs implements Git LFS server storage and authorization.
2//
3// The protocol surface lives in httpd (batch API + basic transfers) and
4// sshd (git-lfs-authenticate); this package owns the pieces both need:
5// content-addressed blob storage behind a small interface, and the
6// short-lived tokens that bridge SSH authentication to the HTTP endpoints.
7//
8// BlobStore is deliberately minimal so an S3-compatible backend is a
9// drop-in: implement the four methods against a bucket and the batch and
10// transfer handlers work unchanged (the server streams as a proxy).
11// Handing clients presigned URLs instead is a later optimization to the
12// batch handler, not a rewrite.
13package lfs
14
15import (
16 "crypto/hmac"
17 "crypto/rand"
18 "crypto/sha256"
19 "encoding/base64"
20 "encoding/hex"
21 "fmt"
22 "io"
23 "io/fs"
24 "os"
25 "path/filepath"
26 "regexp"
27 "strconv"
28 "strings"
29 "time"
30)
31
32// OIDPat is a lowercase sha256 hex digest — the only object name LFS uses.
33var OIDPat = regexp.MustCompile(`^[a-f0-9]{64}$`)
34
35// BlobStore holds LFS objects by their sha256 content address.
36type BlobStore interface {
37 // Put stores the reader's content as oid, verifying both size and
38 // digest; a mismatch stores nothing.
39 Put(oid string, r io.Reader, size int64) error
40 Get(oid string) (io.ReadCloser, int64, error)
41 Exists(oid string) (int64, bool)
42 Delete(oid string) error
43}
44
45// LocalStore is the on-disk backend: <root>/<aa>/<bb>/<oid>, written via a
46// temp file and renamed only after the digest checks out.
47type LocalStore struct {
48 Root string
49}
50
51func (s LocalStore) path(oid string) string {
52 return filepath.Join(s.Root, oid[:2], oid[2:4], oid)
53}
54
55func (s LocalStore) Put(oid string, r io.Reader, size int64) error {
56 if !OIDPat.MatchString(oid) {
57 return fmt.Errorf("bad oid %q", oid)
58 }
59 dir := filepath.Dir(s.path(oid))
60 if err := os.MkdirAll(dir, 0o755); err != nil {
61 return err
62 }
63 tmp, err := os.CreateTemp(dir, ".upload-*")
64 if err != nil {
65 return err
66 }
67 defer func() {
68 tmp.Close()
69 os.Remove(tmp.Name())
70 }()
71 h := sha256.New()
72 n, err := io.Copy(io.MultiWriter(tmp, h), io.LimitReader(r, size+1))
73 if err != nil {
74 return err
75 }
76 if n != size {
77 return fmt.Errorf("size mismatch: got %d bytes, expected %d", n, size)
78 }
79 if sum := hex.EncodeToString(h.Sum(nil)); sum != oid {
80 return fmt.Errorf("content digest %s does not match oid", sum[:12])
81 }
82 if err := tmp.Close(); err != nil {
83 return err
84 }
85 return os.Rename(tmp.Name(), s.path(oid))
86}
87
88func (s LocalStore) Get(oid string) (io.ReadCloser, int64, error) {
89 if !OIDPat.MatchString(oid) {
90 return nil, 0, fmt.Errorf("bad oid %q", oid)
91 }
92 f, err := os.Open(s.path(oid))
93 if err != nil {
94 return nil, 0, err
95 }
96 fi, err := f.Stat()
97 if err != nil {
98 f.Close()
99 return nil, 0, err
100 }
101 return f, fi.Size(), nil
102}
103
104func (s LocalStore) Exists(oid string) (int64, bool) {
105 if !OIDPat.MatchString(oid) {
106 return 0, false
107 }
108 fi, err := os.Stat(s.path(oid))
109 if err != nil {
110 return 0, false
111 }
112 return fi.Size(), true
113}
114
115func (s LocalStore) Delete(oid string) error {
116 if !OIDPat.MatchString(oid) {
117 return fmt.Errorf("bad oid %q", oid)
118 }
119 return os.Remove(s.path(oid))
120}
121
122// Tokens bridge SSH authentication to the HTTP endpoints: stateless,
123// HMAC-signed, scoped to one repo and one operation, short-lived, and
124// bound to the SSH key that obtained them, which must still be live
125// when the token is used (#285). The secret persists in the settings
126// table so tokens survive restarts.
127
128const TokenTTL = time.Hour
129
130// Sign mints a token for op ("download" or "upload") on repoID, bound
131// to keyID: the SSH key, user or deploy, that asked for it, or 0 for an
132// anonymous download of a public repository. fingerprint is that key's
133// fingerprint, "" for key 0. SQLite reuses the id of a deleted key, so
134// the token carries a hash of the fingerprint as well and a new key
135// given the old id does not inherit the old key's tokens (#303).
136func Sign(secret []byte, repoID, keyID int64, fingerprint, op string, now time.Time) string {
137 payload := fmt.Sprintf("%d:%d:%s:%s:%d", repoID, keyID, KeyPin(fingerprint), op, now.Add(TokenTTL).Unix())
138 mac := hmac.New(sha256.New, secret)
139 mac.Write([]byte(payload))
140 return base64.RawURLEncoding.EncodeToString([]byte(payload)) + "." +
141 base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
142}
143
144// KeyPin is the fingerprint's form in a token: the first 16 hex
145// characters of its SHA-256, or "" for no key.
146func KeyPin(fingerprint string) string {
147 if fingerprint == "" {
148 return ""
149 }
150 sum := sha256.Sum256([]byte(fingerprint))
151 return hex.EncodeToString(sum[:8])
152}
153
154// Grant is what a verified token authorizes.
155type Grant struct {
156 RepoID int64
157 KeyID int64 // 0: an anonymous download of a public repository
158 KeyPin string // KeyPin of the key's fingerprint; "" when KeyID is 0
159 Op string
160}
161
162// Verify checks a token's MAC, shape and expiry. A token from before
163// tokens named their key, or before they carried its fingerprint, does
164// not verify.
165func Verify(secret []byte, token string, now time.Time) (Grant, bool) {
166 payloadB64, macB64, found := strings.Cut(token, ".")
167 if !found {
168 return Grant{}, false
169 }
170 payload, err := base64.RawURLEncoding.DecodeString(payloadB64)
171 if err != nil {
172 return Grant{}, false
173 }
174 gotMAC, err := base64.RawURLEncoding.DecodeString(macB64)
175 if err != nil {
176 return Grant{}, false
177 }
178 mac := hmac.New(sha256.New, secret)
179 mac.Write(payload)
180 if !hmac.Equal(mac.Sum(nil), gotMAC) {
181 return Grant{}, false
182 }
183 parts := strings.Split(string(payload), ":")
184 if len(parts) != 5 {
185 return Grant{}, false
186 }
187 repoID, err1 := strconv.ParseInt(parts[0], 10, 64)
188 keyID, err2 := strconv.ParseInt(parts[1], 10, 64)
189 exp, err3 := strconv.ParseInt(parts[4], 10, 64)
190 if err1 != nil || err2 != nil || err3 != nil || keyID < 0 || now.Unix() > exp {
191 return Grant{}, false
192 }
193 if (keyID == 0) != (parts[2] == "") {
194 return Grant{}, false
195 }
196 if parts[3] != "download" && parts[3] != "upload" {
197 return Grant{}, false
198 }
199 return Grant{RepoID: repoID, KeyID: keyID, KeyPin: parts[2], Op: parts[3]}, true
200}
201
202// NewSecret returns 32 random bytes, hex-encoded for the settings table.
203func NewSecret() string {
204 buf := make([]byte, 32)
205 rand.Read(buf)
206 return hex.EncodeToString(buf)
207}
208
209// Orphans lists objects in the store that no repository references and
210// that are older than minAge: an object uploaded ahead of the push that
211// will reference it is not an orphan yet. referenced holds the object ids
212// every repository's pointers name.
213func (s LocalStore) Orphans(referenced map[string]bool, minAge time.Duration) ([]Orphan, error) {
214 cutoff := time.Now().Add(-minAge)
215 var out []Orphan
216 err := filepath.WalkDir(s.Root, func(path string, d fs.DirEntry, err error) error {
217 if err != nil || d.IsDir() {
218 return nil
219 }
220 oid := d.Name()
221 if !OIDPat.MatchString(oid) || referenced[oid] {
222 return nil
223 }
224 info, err := d.Info()
225 if err != nil || info.ModTime().After(cutoff) {
226 return nil
227 }
228 out = append(out, Orphan{OID: oid, Size: info.Size()})
229 return nil
230 })
231 return out, err
232}
233
234// Orphan is one unreferenced object.
235type Orphan struct {
236 OID string
237 Size int64
238}
239
240// Size sums every object in the store.
241func (s LocalStore) Size() int64 {
242 var total int64
243 filepath.WalkDir(s.Root, func(_ string, d fs.DirEntry, err error) error {
244 if err == nil && !d.IsDir() {
245 if fi, err := d.Info(); err == nil {
246 total += fi.Size()
247 }
248 }
249 return nil
250 })
251 return total
252}
253
254// RootFor is the store root a configuration implies.
255func RootFor(lfsRoot, serverRoot string) string {
256 if lfsRoot != "" {
257 return lfsRoot
258 }
259 return filepath.Join(serverRoot, "lfs")
260}