internal/control/migrate.go
350 lines · 12721 bytes
1package control
2
3import (
4 "encoding/json"
5 "fmt"
6 "io"
7 "strconv"
8 "strings"
9
10 "gitbay.org/gitbay/internal/gitutil"
11 "gitbay.org/gitbay/internal/protocol"
12 "gitbay.org/gitbay/internal/store"
13)
14
15func init() {
16 register(Command{Path: []string{"account", "export"},
17 Summary: "write your account bundle (profile, repos, issues, MRs) as JSON",
18 Usage: "account export > bundle.json",
19 ReadOnly: true, Run: runAccountExport})
20 register(Command{Path: []string{"account", "import-bundle"},
21 Summary: "replay an account bundle (see gitbay migrate)",
22 Usage: "account import-bundle [--source <host>] < bundle.json",
23 ReadsStdin: true, Run: runAccountImportBundle})
24}
25
26// The bundle format doubles as a user-level backup. Keys are never
27// exported (trust is per-instance); emails arrive unverified.
28const bundleVersion = "gitbay-account/1"
29
30type bundleComment struct {
31 Author string `json:"author"`
32 Body string `json:"body"`
33 CreatedAt string `json:"created_at"`
34}
35type bundleIssue struct {
36 Number int64 `json:"number"`
37 Title string `json:"title"`
38 Body string `json:"body"`
39 State string `json:"state"`
40 Author string `json:"author"`
41 CreatedAt string `json:"created_at"`
42 Labels []string `json:"labels,omitempty"`
43 Comments []bundleComment `json:"comments,omitempty"`
44}
45type bundleMR struct {
46 Number int64 `json:"number"`
47 Title string `json:"title"`
48 Body string `json:"body"`
49 State string `json:"state"`
50 Author string `json:"author"`
51 SourceRef string `json:"source_ref"`
52 TargetRef string `json:"target_ref"`
53 // HeadSHA and MergedBase are what a diff is measured between. The
54 // bundle carried neither, so every migrated merge request arrived
55 // with an empty head and `mr diff` could only fail on it (#128).
56 // They are recorded whether or not the objects have been pushed yet;
57 // the ref is pointed at the head once they have.
58 HeadSHA string `json:"head_sha,omitempty"`
59 MergedBase string `json:"merged_base,omitempty"`
60 MergedAt string `json:"merged_at,omitempty"`
61 CreatedAt string `json:"created_at"`
62 Comments []bundleComment `json:"comments,omitempty"`
63}
64type bundleRepo struct {
65 Name string `json:"name"`
66 Visibility string `json:"visibility"`
67 DefaultBranch string `json:"default_branch"`
68 Description string `json:"description,omitempty"`
69 Topics []string `json:"topics,omitempty"`
70 Settings store.RepoSettings `json:"settings"`
71 Issues []bundleIssue `json:"issues,omitempty"`
72 MRs []bundleMR `json:"mrs,omitempty"`
73}
74type bundle struct {
75 Bundle string `json:"bundle"`
76 Username string `json:"username"`
77 Profile store.Profile `json:"profile"`
78 Emails []string `json:"emails,omitempty"`
79 Repos []bundleRepo `json:"repos"`
80}
81
82func runAccountExport(c *Ctx, args []string) int {
83 if len(args) != 0 {
84 return c.usage()
85 }
86 b := bundle{Bundle: bundleVersion, Username: c.User.Username}
87 b.Profile, _ = c.Store.OwnerProfile("user", c.User.ID)
88 b.Emails, _ = c.Store.UserEmailAddresses(c.User.ID)
89 repos, err := c.Store.ListReposForOwner("user", c.User.ID)
90 if err != nil {
91 return c.fail(protocol.ExitFailure, "%v", err)
92 }
93 for _, r := range repos {
94 br := bundleRepo{
95 Name: r.Name,
96 Visibility: r.Visibility,
97 DefaultBranch: r.DefaultBranch,
98 Description: gitutil.ReadDescription(RepoDir(c.Cfg.Server.Root, r.OwnerName, r.Name)),
99 Settings: r.Settings,
100 }
101 br.Topics, _ = c.Store.ListTopics(r.ID)
102 issues, _ := c.Store.ListIssues(r.ID, "all", 0, 0)
103 for i := len(issues) - 1; i >= 0; i-- { // ascending numbers
104 iss := issues[i]
105 full, err := c.Store.IssueByNumber(r.ID, iss.Number)
106 if err != nil {
107 continue
108 }
109 bi := bundleIssue{Number: full.Number, Title: full.Title, Body: full.Body,
110 State: full.State, Author: full.Author, CreatedAt: full.CreatedAt, Labels: full.Labels}
111 if cs, err := c.Store.ListIssueComments(full.ID); err == nil {
112 for _, cm := range cs {
113 bi.Comments = append(bi.Comments, bundleComment{cm.Author, cm.Body, cm.CreatedAt})
114 }
115 }
116 br.Issues = append(br.Issues, bi)
117 }
118 mrs, _ := c.Store.ListMRs(r.ID, "all", 0, 0)
119 for i := len(mrs) - 1; i >= 0; i-- {
120 m := mrs[i]
121 bm := bundleMR{Number: m.Number, Title: m.Title, Body: m.Body, State: m.State,
122 Author: m.Author, SourceRef: m.SourceRef, TargetRef: m.TargetRef,
123 HeadSHA: m.HeadSHA, MergedBase: m.MergedBase, MergedAt: m.MergedAt,
124 CreatedAt: m.CreatedAt}
125 if cs, err := c.Store.ListMRComments(m.ID); err == nil {
126 for _, cm := range cs {
127 bm.Comments = append(bm.Comments, bundleComment{cm.Author, cm.Body, cm.CreatedAt})
128 }
129 }
130 br.MRs = append(br.MRs, bm)
131 }
132 b.Repos = append(b.Repos, br)
133 }
134 enc := json.NewEncoder(c.Stdout)
135 enc.SetIndent("", " ")
136 if err := enc.Encode(b); err != nil {
137 return protocol.ExitFailure
138 }
139 return protocol.ExitOK
140}
141
142func migAttribution(src, kind, author, date string, n int64) string {
143 return fmt.Sprintf("> migrated %s %s#%d — %s, %.10s\n\n", kind, src, n, author, date)
144}
145
146// runAccountImportBundle replays a bundle under the calling user. Repos are
147// created empty (the CLI pushes git data with the user's own key); issues,
148// MRs, and comments arrive attributed inline. Push-blocking policies
149// (require_signed_commits, protected_branches) are deferred and reported so
150// the git push that follows cannot be refused by them. Resumable: markers
151// skip everything already imported.
152func runAccountImportBundle(c *Ctx, args []string) int {
153 f, err := parseFlags(args, flagSpec{Values: []string{"--source"}, MaxPos: 0, Usage: "account import-bundle [--source <host>] < bundle.json"})
154 if err != nil {
155 return c.fail(protocol.ExitUsage, "%v", err)
156 }
157 src := f.Value("--source")
158 if src == "" {
159 src = "the previous instance"
160 }
161 var b bundle
162 if err := json.NewDecoder(io.LimitReader(c.Stdin, 512<<20)).Decode(&b); err != nil {
163 return c.fail(protocol.ExitUsage, "bundle does not parse: %v", err)
164 }
165 if b.Bundle != bundleVersion {
166 return c.fail(protocol.ExitUsage, "unsupported bundle %q (want %s)", b.Bundle, bundleVersion)
167 }
168
169 if b.Profile.Description != "" || b.Profile.Website != "" || len(b.Profile.Links) > 0 {
170 c.Store.SetOwnerProfile("user", c.User.ID, b.Profile)
171 }
172 for _, addr := range b.Emails {
173 c.Store.AddEmail(c.User.ID, addr, "", false) // unverified; re-verify here
174 }
175
176 type deferred struct {
177 Repo string `json:"repo"`
178 Commands []string `json:"commands"`
179 }
180 var repos, issues, mrs, comments, skipped int
181 var deferrals []deferred
182 for _, br := range b.Repos {
183 path := c.User.Username + "/" + br.Name
184 repo, err := c.Store.RepoByPath(path)
185 if err != nil {
186 id, err := c.Store.CreateRepo("user", c.User.ID, br.Name, br.Visibility)
187 if err != nil {
188 return c.fail(protocol.ExitFailure, "creating %s: %v", path, err)
189 }
190 dir := RepoDir(c.Cfg.Server.Root, c.User.Username, br.Name)
191 branch := br.DefaultBranch
192 if branch == "" {
193 branch = "main"
194 }
195 if err := gitutil.InitBare(dir, branch, HooksDir(c.Cfg.Server.Root)); err != nil {
196 c.Store.DeleteRepo(id)
197 return c.fail(protocol.ExitFailure, "initializing %s: %v", path, err)
198 }
199 if br.Description != "" {
200 gitutil.WriteDescription(dir, br.Description)
201 }
202 repo, err = c.Store.RepoByPath(path)
203 if err != nil {
204 return c.fail(protocol.ExitFailure, "%v", err)
205 }
206 repos++
207 }
208 for _, tpc := range br.Topics {
209 c.Store.AddTopic(repo.ID, tpc)
210 }
211 // Settings minus the two that would refuse the git push coming
212 // right after this; the user re-applies them once data is in.
213 s := br.Settings
214 var cmds []string
215 if s.RequireSignedCommits {
216 cmds = append(cmds, fmt.Sprintf("gitbay repo settings require-signed %s on", path))
217 s.RequireSignedCommits = false
218 }
219 if len(s.ProtectedBranches) > 0 {
220 for _, pb := range s.ProtectedBranches {
221 cmds = append(cmds, fmt.Sprintf("gitbay repo settings protect %s %s", path, pb))
222 }
223 s.ProtectedBranches = nil
224 }
225 if len(cmds) > 0 {
226 deferrals = append(deferrals, deferred{path, cmds})
227 }
228 // An import writes the whole blob: the repository was created a
229 // few lines up and nobody else holds settings on it yet.
230 s.GitDaemon = false // instance-dependent; opt back in explicitly
231 c.Store.UpdateRepoSettings(repo.ID, func(cur *store.RepoSettings) { *cur = s })
232
233 for _, bi := range br.Issues {
234 key := fmt.Sprintf("mig-issue:%d", bi.Number)
235 if _, seen, _ := c.Store.ImportMarker(repo.ID, key); seen {
236 skipped++
237 continue
238 }
239 body := migAttribution(src, "issue", bi.Author, bi.CreatedAt, bi.Number) + bi.Body
240 n, err := c.Store.CreateIssue(repo.ID, c.User.ID, bi.Title, body, "md")
241 if err != nil {
242 return c.fail(protocol.ExitFailure, "%v", err)
243 }
244 iss, err := c.Store.IssueByNumber(repo.ID, n)
245 if err != nil {
246 return c.fail(protocol.ExitFailure, "%v", err)
247 }
248 for _, l := range bi.Labels {
249 c.Store.SetIssueLabel(repo, iss.ID, l, true)
250 }
251 if bi.State != "open" {
252 c.Store.SetIssueState(iss.ID, "closed")
253 }
254 for _, cm := range bi.Comments {
255 c.Store.AddIssueComment(iss.ID, c.User.ID,
256 fmt.Sprintf("> %s, %.10s\n\n%s", cm.Author, cm.CreatedAt, cm.Body), "md")
257 comments++
258 }
259 c.Store.SetImportMarker(repo.ID, key, fmt.Sprint(n))
260 issues++
261 }
262 for _, bm := range br.MRs {
263 key := fmt.Sprintf("mig-mr:%d", bm.Number)
264 if val, seen, _ := c.Store.ImportMarker(repo.ID, key); seen {
265 skipped++
266 // A bundle is imported before the git push as often as
267 // after, so the objects a merge request needs may only
268 // have arrived since. Re-running the import is how the
269 // head ref gets set, and doing that costs nothing when it
270 // is already right.
271 if local, err := strconv.ParseInt(val, 10, 64); err == nil {
272 setMigratedHead(c, repo, local, bm.HeadSHA)
273 }
274 continue
275 }
276 body := migAttribution(src, "merge request", bm.Author, bm.CreatedAt, bm.Number) + bm.Body
277 n, err := c.Store.CreateMR(repo.ID, c.User.ID, repo.ID, bm.SourceRef, bm.TargetRef, bm.Title, body, bm.HeadSHA, "md", false)
278 if err != nil {
279 return c.fail(protocol.ExitFailure, "%v", err)
280 }
281 mr, err := c.Store.MRByNumber(repo.ID, n)
282 if err != nil {
283 return c.fail(protocol.ExitFailure, "%v", err)
284 }
285 switch {
286 case bm.State == "merged":
287 // MarkMerged, not SetMRState: a merged MR's diff is
288 // measured from the base recorded at merge time, and
289 // SetMRState leaves that empty.
290 c.Store.MarkMerged(mr.ID, bm.MergedBase, 0, bm.MergedAt)
291 case bm.State != "open":
292 state := bm.State
293 if state == "source_gone" {
294 state = "closed"
295 }
296 c.Store.SetMRState(mr.ID, state)
297 }
298 setMigratedHead(c, repo, n, bm.HeadSHA)
299 for _, cm := range bm.Comments {
300 c.Store.AddMRComment(mr.ID, c.User.ID,
301 fmt.Sprintf("> %s, %.10s\n\n%s", cm.Author, cm.CreatedAt, cm.Body), "md")
302 comments++
303 }
304 c.Store.SetImportMarker(repo.ID, key, fmt.Sprint(n))
305 mrs++
306 }
307 }
308 d := map[string]any{
309 "repos": repos, "issues": issues, "mrs": mrs, "comments": comments,
310 "already_imported": skipped, "deferred_settings": deferrals,
311 }
312 return c.emit(d, func(w io.Writer) {
313 fmt.Fprintf(w, "imported %d repos, %d issues, %d MRs, %d comments (%d already present)\n",
314 repos, issues, mrs, comments, skipped)
315 if len(deferrals) > 0 {
316 fmt.Fprintf(w, "\nafter pushing git data, re-apply the deferred policies:\n")
317 for _, df := range deferrals {
318 for _, cmd := range df.Commands {
319 fmt.Fprintf(w, " %s\n", cmd)
320 }
321 }
322 }
323 })
324}
325
326var _ = strings.TrimSpace // placeholder against accidental import drops
327
328// setMigratedHead points refs/merge-requests/<n>/head at the head a
329// bundle recorded, once the objects for it are present. `mr diff`
330// resolves through that ref, not through the stored head_sha, so a merge
331// request whose source branch is gone — every merged one — has nothing to
332// diff without it (#128).
333//
334// Best-effort by design: a bundle is imported before the git push as
335// often as after. head_sha is stored either way, and running the import
336// again after the push sets the ref.
337func setMigratedHead(c *Ctx, repo store.Repo, number int64, headSHA string) {
338 if headSHA == "" {
339 return
340 }
341 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
342 if !gitutil.HasCommit(dir, headSHA) {
343 return
344 }
345 ref := fmt.Sprintf("refs/merge-requests/%d/head", number)
346 if cur, err := gitutil.ResolveRef(dir, ref); err == nil && cur == headSHA {
347 return
348 }
349 gitutil.UpdateRefCAS(dir, ref, headSHA, "")
350}