internal/control/migrate.go
355 lines · 12953 bytes
1package control
2
3import (
4 "encoding/json"
5 "fmt"
6 "io"
7 "strconv"
8 "strings"
9
10 "gitbay.org/gitbay/internal/gitutil"
11 "gitbay.org/gitbay/internal/protocol"
12 "gitbay.org/gitbay/internal/store"
13)
14
15func init() {
16 register(Command{Path: []string{"account", "export"},
17 Summary: "write your account bundle (profile, repos, issues, MRs) as JSON",
18 Usage: "account export > bundle.json",
19 Examples: []string{"account export > bundle.json"},
20 ReadOnly: true, Run: runAccountExport})
21 register(Command{Path: []string{"account", "import-bundle"},
22 Summary: "replay an account bundle (see gitbay migrate)",
23 Usage: "account import-bundle [--source <host>] < bundle.json",
24 Flags: []Flag{
25 {"--source", "<host>", "the previous instance, for attribution", ""},
26 },
27 Examples: []string{"account import-bundle --source old.example.org < bundle.json"},
28 ReadsStdin: true, Run: runAccountImportBundle})
29}
30
31// The bundle format doubles as a user-level backup. Keys are never
32// exported (trust is per-instance); emails arrive unverified.
33const bundleVersion = "gitbay-account/1"
34
35type bundleComment struct {
36 Author string `json:"author"`
37 Body string `json:"body"`
38 CreatedAt string `json:"created_at"`
39}
40type bundleIssue struct {
41 Number int64 `json:"number"`
42 Title string `json:"title"`
43 Body string `json:"body"`
44 State string `json:"state"`
45 Author string `json:"author"`
46 CreatedAt string `json:"created_at"`
47 Labels []string `json:"labels,omitempty"`
48 Comments []bundleComment `json:"comments,omitempty"`
49}
50type bundleMR struct {
51 Number int64 `json:"number"`
52 Title string `json:"title"`
53 Body string `json:"body"`
54 State string `json:"state"`
55 Author string `json:"author"`
56 SourceRef string `json:"source_ref"`
57 TargetRef string `json:"target_ref"`
58 // HeadSHA and MergedBase are what a diff is measured between. The
59 // bundle carried neither, so every migrated merge request arrived
60 // with an empty head and `mr diff` could only fail on it (#128).
61 // They are recorded whether or not the objects have been pushed yet;
62 // the ref is pointed at the head once they have.
63 HeadSHA string `json:"head_sha,omitempty"`
64 MergedBase string `json:"merged_base,omitempty"`
65 MergedAt string `json:"merged_at,omitempty"`
66 CreatedAt string `json:"created_at"`
67 Comments []bundleComment `json:"comments,omitempty"`
68}
69type bundleRepo struct {
70 Name string `json:"name"`
71 Visibility string `json:"visibility"`
72 DefaultBranch string `json:"default_branch"`
73 Description string `json:"description,omitempty"`
74 Topics []string `json:"topics,omitempty"`
75 Settings store.RepoSettings `json:"settings"`
76 Issues []bundleIssue `json:"issues,omitempty"`
77 MRs []bundleMR `json:"mrs,omitempty"`
78}
79type bundle struct {
80 Bundle string `json:"bundle"`
81 Username string `json:"username"`
82 Profile store.Profile `json:"profile"`
83 Emails []string `json:"emails,omitempty"`
84 Repos []bundleRepo `json:"repos"`
85}
86
87func runAccountExport(c *Ctx, args []string) int {
88 if len(args) != 0 {
89 return c.usage()
90 }
91 b := bundle{Bundle: bundleVersion, Username: c.User.Username}
92 b.Profile, _ = c.Store.OwnerProfile("user", c.User.ID)
93 b.Emails, _ = c.Store.UserEmailAddresses(c.User.ID)
94 repos, err := c.Store.ListReposForOwner("user", c.User.ID)
95 if err != nil {
96 return c.fail(protocol.ExitFailure, "%v", err)
97 }
98 for _, r := range repos {
99 br := bundleRepo{
100 Name: r.Name,
101 Visibility: r.Visibility,
102 DefaultBranch: r.DefaultBranch,
103 Description: gitutil.ReadDescription(RepoDir(c.Cfg.Server.Root, r.OwnerName, r.Name)),
104 Settings: r.Settings,
105 }
106 br.Topics, _ = c.Store.ListTopics(r.ID)
107 issues, _ := c.Store.ListIssues(r.ID, "all", 0, 0)
108 for i := len(issues) - 1; i >= 0; i-- { // ascending numbers
109 iss := issues[i]
110 full, err := c.Store.IssueByNumber(r.ID, iss.Number)
111 if err != nil {
112 continue
113 }
114 bi := bundleIssue{Number: full.Number, Title: full.Title, Body: full.Body,
115 State: full.State, Author: full.Author, CreatedAt: full.CreatedAt, Labels: full.Labels}
116 if cs, err := c.Store.ListIssueComments(full.ID); err == nil {
117 for _, cm := range cs {
118 bi.Comments = append(bi.Comments, bundleComment{cm.Author, cm.Body, cm.CreatedAt})
119 }
120 }
121 br.Issues = append(br.Issues, bi)
122 }
123 mrs, _ := c.Store.ListMRs(r.ID, "all", 0, 0)
124 for i := len(mrs) - 1; i >= 0; i-- {
125 m := mrs[i]
126 bm := bundleMR{Number: m.Number, Title: m.Title, Body: m.Body, State: m.State,
127 Author: m.Author, SourceRef: m.SourceRef, TargetRef: m.TargetRef,
128 HeadSHA: m.HeadSHA, MergedBase: m.MergedBase, MergedAt: m.MergedAt,
129 CreatedAt: m.CreatedAt}
130 if cs, err := c.Store.ListMRComments(m.ID); err == nil {
131 for _, cm := range cs {
132 bm.Comments = append(bm.Comments, bundleComment{cm.Author, cm.Body, cm.CreatedAt})
133 }
134 }
135 br.MRs = append(br.MRs, bm)
136 }
137 b.Repos = append(b.Repos, br)
138 }
139 enc := json.NewEncoder(c.Stdout)
140 enc.SetIndent("", " ")
141 if err := enc.Encode(b); err != nil {
142 return protocol.ExitFailure
143 }
144 return protocol.ExitOK
145}
146
147func migAttribution(src, kind, author, date string, n int64) string {
148 return fmt.Sprintf("> migrated %s %s#%d — %s, %.10s\n\n", kind, src, n, author, date)
149}
150
151// runAccountImportBundle replays a bundle under the calling user. Repos are
152// created empty (the CLI pushes git data with the user's own key); issues,
153// MRs, and comments arrive attributed inline. Push-blocking policies
154// (require_signed_commits, protected_branches) are deferred and reported so
155// the git push that follows cannot be refused by them. Resumable: markers
156// skip everything already imported.
157func runAccountImportBundle(c *Ctx, args []string) int {
158 f, err := c.parseArgs(args, flagSpec{Values: []string{"--source"}, MaxPos: 0, Usage: "account import-bundle [--source <host>] < bundle.json"})
159 if err != nil {
160 return c.fail(protocol.ExitUsage, "%v", err)
161 }
162 src := f.Value("--source")
163 if src == "" {
164 src = "the previous instance"
165 }
166 var b bundle
167 if err := json.NewDecoder(io.LimitReader(c.Stdin, 512<<20)).Decode(&b); err != nil {
168 return c.fail(protocol.ExitUsage, "bundle does not parse: %v", err)
169 }
170 if b.Bundle != bundleVersion {
171 return c.fail(protocol.ExitUsage, "unsupported bundle %q (want %s)", b.Bundle, bundleVersion)
172 }
173
174 if b.Profile.Description != "" || b.Profile.Website != "" || len(b.Profile.Links) > 0 {
175 c.Store.SetOwnerProfile("user", c.User.ID, b.Profile)
176 }
177 for _, addr := range b.Emails {
178 c.Store.AddEmail(c.User.ID, addr, "", false) // unverified; re-verify here
179 }
180
181 type deferred struct {
182 Repo string `json:"repo"`
183 Commands []string `json:"commands"`
184 }
185 var repos, issues, mrs, comments, skipped int
186 var deferrals []deferred
187 for _, br := range b.Repos {
188 path := c.User.Username + "/" + br.Name
189 repo, err := c.Store.RepoByPath(path)
190 if err != nil {
191 id, err := c.Store.CreateRepo("user", c.User.ID, br.Name, br.Visibility)
192 if err != nil {
193 return c.fail(protocol.ExitFailure, "creating %s: %v", path, err)
194 }
195 dir := RepoDir(c.Cfg.Server.Root, c.User.Username, br.Name)
196 branch := br.DefaultBranch
197 if branch == "" {
198 branch = "main"
199 }
200 if err := gitutil.InitBare(dir, branch, HooksDir(c.Cfg.Server.Root)); err != nil {
201 c.Store.DeleteRepo(id)
202 return c.fail(protocol.ExitFailure, "initializing %s: %v", path, err)
203 }
204 if br.Description != "" {
205 gitutil.WriteDescription(dir, br.Description)
206 }
207 repo, err = c.Store.RepoByPath(path)
208 if err != nil {
209 return c.fail(protocol.ExitFailure, "%v", err)
210 }
211 repos++
212 }
213 for _, tpc := range br.Topics {
214 c.Store.AddTopic(repo.ID, tpc)
215 }
216 // Settings minus the two that would refuse the git push coming
217 // right after this; the user re-applies them once data is in.
218 s := br.Settings
219 var cmds []string
220 if s.RequireSignedCommits {
221 cmds = append(cmds, fmt.Sprintf("gitbay repo settings require-signed %s on", path))
222 s.RequireSignedCommits = false
223 }
224 if len(s.ProtectedBranches) > 0 {
225 for _, pb := range s.ProtectedBranches {
226 cmds = append(cmds, fmt.Sprintf("gitbay repo settings protect %s %s", path, pb))
227 }
228 s.ProtectedBranches = nil
229 }
230 if len(cmds) > 0 {
231 deferrals = append(deferrals, deferred{path, cmds})
232 }
233 // An import writes the whole blob: the repository was created a
234 // few lines up and nobody else holds settings on it yet.
235 s.GitDaemon = false // instance-dependent; opt back in explicitly
236 c.Store.UpdateRepoSettings(repo.ID, func(cur *store.RepoSettings) { *cur = s })
237
238 for _, bi := range br.Issues {
239 key := fmt.Sprintf("mig-issue:%d", bi.Number)
240 if _, seen, _ := c.Store.ImportMarker(repo.ID, key); seen {
241 skipped++
242 continue
243 }
244 body := migAttribution(src, "issue", bi.Author, bi.CreatedAt, bi.Number) + bi.Body
245 n, err := c.Store.CreateIssue(repo.ID, c.User.ID, bi.Title, body, "md")
246 if err != nil {
247 return c.fail(protocol.ExitFailure, "%v", err)
248 }
249 iss, err := c.Store.IssueByNumber(repo.ID, n)
250 if err != nil {
251 return c.fail(protocol.ExitFailure, "%v", err)
252 }
253 for _, l := range bi.Labels {
254 c.Store.SetIssueLabel(repo, iss.ID, l, true)
255 }
256 if bi.State != "open" {
257 c.Store.SetIssueState(iss.ID, "closed")
258 }
259 for _, cm := range bi.Comments {
260 c.Store.AddIssueComment(iss.ID, c.User.ID,
261 fmt.Sprintf("> %s, %.10s\n\n%s", cm.Author, cm.CreatedAt, cm.Body), "md")
262 comments++
263 }
264 c.Store.SetImportMarker(repo.ID, key, fmt.Sprint(n))
265 issues++
266 }
267 for _, bm := range br.MRs {
268 key := fmt.Sprintf("mig-mr:%d", bm.Number)
269 if val, seen, _ := c.Store.ImportMarker(repo.ID, key); seen {
270 skipped++
271 // A bundle is imported before the git push as often as
272 // after, so the objects a merge request needs may only
273 // have arrived since. Re-running the import is how the
274 // head ref gets set, and doing that costs nothing when it
275 // is already right.
276 if local, err := strconv.ParseInt(val, 10, 64); err == nil {
277 setMigratedHead(c, repo, local, bm.HeadSHA)
278 }
279 continue
280 }
281 body := migAttribution(src, "merge request", bm.Author, bm.CreatedAt, bm.Number) + bm.Body
282 n, err := c.Store.CreateMR(repo.ID, c.User.ID, repo.ID, bm.SourceRef, bm.TargetRef, bm.Title, body, bm.HeadSHA, "md", false)
283 if err != nil {
284 return c.fail(protocol.ExitFailure, "%v", err)
285 }
286 mr, err := c.Store.MRByNumber(repo.ID, n)
287 if err != nil {
288 return c.fail(protocol.ExitFailure, "%v", err)
289 }
290 switch {
291 case bm.State == "merged":
292 // MarkMerged, not SetMRState: a merged MR's diff is
293 // measured from the base recorded at merge time, and
294 // SetMRState leaves that empty.
295 c.Store.MarkMerged(mr.ID, bm.MergedBase, 0, bm.MergedAt)
296 case bm.State != "open":
297 state := bm.State
298 if state == "source_gone" {
299 state = "closed"
300 }
301 c.Store.SetMRState(mr.ID, state)
302 }
303 setMigratedHead(c, repo, n, bm.HeadSHA)
304 for _, cm := range bm.Comments {
305 c.Store.AddMRComment(mr.ID, c.User.ID,
306 fmt.Sprintf("> %s, %.10s\n\n%s", cm.Author, cm.CreatedAt, cm.Body), "md")
307 comments++
308 }
309 c.Store.SetImportMarker(repo.ID, key, fmt.Sprint(n))
310 mrs++
311 }
312 }
313 d := map[string]any{
314 "repos": repos, "issues": issues, "mrs": mrs, "comments": comments,
315 "already_imported": skipped, "deferred_settings": deferrals,
316 }
317 return c.emit(d, func(w io.Writer) {
318 fmt.Fprintf(w, "imported %d repos, %d issues, %d MRs, %d comments (%d already present)\n",
319 repos, issues, mrs, comments, skipped)
320 if len(deferrals) > 0 {
321 fmt.Fprintf(w, "\nafter pushing git data, re-apply the deferred policies:\n")
322 for _, df := range deferrals {
323 for _, cmd := range df.Commands {
324 fmt.Fprintf(w, " %s\n", cmd)
325 }
326 }
327 }
328 })
329}
330
331var _ = strings.TrimSpace // placeholder against accidental import drops
332
333// setMigratedHead points refs/merge-requests/<n>/head at the head a
334// bundle recorded, once the objects for it are present. `mr diff`
335// resolves through that ref, not through the stored head_sha, so a merge
336// request whose source branch is gone — every merged one — has nothing to
337// diff without it (#128).
338//
339// Best-effort by design: a bundle is imported before the git push as
340// often as after. head_sha is stored either way, and running the import
341// again after the push sets the ref.
342func setMigratedHead(c *Ctx, repo store.Repo, number int64, headSHA string) {
343 if headSHA == "" {
344 return
345 }
346 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
347 if !gitutil.HasCommit(dir, headSHA) {
348 return
349 }
350 ref := fmt.Sprintf("refs/merge-requests/%d/head", number)
351 if cur, err := gitutil.ResolveRef(dir, ref); err == nil && cur == headSHA {
352 return
353 }
354 gitutil.UpdateRefCAS(dir, ref, headSHA, "")
355}