internal/control/migrate.go
351 lines · 12804 bytes
1package control
2
3import (
4 "encoding/json"
5 "fmt"
6 "io"
7 "strconv"
8 "strings"
9
10 "gitbay.org/gitbay/internal/gitutil"
11 "gitbay.org/gitbay/internal/protocol"
12 "gitbay.org/gitbay/internal/store"
13)
14
15func init() {
16 register(Command{Path: []string{"account", "export"},
17 Summary: "write your account bundle (profile, repos, issues, MRs) as JSON",
18 Usage: "account export > bundle.json",
19 ReadOnly: true, Run: runAccountExport})
20 register(Command{Path: []string{"account", "import-bundle"},
21 Summary: "replay an account bundle (see gitbay migrate)",
22 Usage: "account import-bundle [--source <host>] < bundle.json",
23 ReadsStdin: true, Run: runAccountImportBundle})
24}
25
26// The bundle format doubles as a user-level backup. Keys are never
27// exported (trust is per-instance); emails arrive unverified.
28const bundleVersion = "gitbay-account/1"
29
30type bundleComment struct {
31 Author string `json:"author"`
32 Body string `json:"body"`
33 CreatedAt string `json:"created_at"`
34}
35type bundleIssue struct {
36 Number int64 `json:"number"`
37 Title string `json:"title"`
38 Body string `json:"body"`
39 State string `json:"state"`
40 Author string `json:"author"`
41 CreatedAt string `json:"created_at"`
42 Labels []string `json:"labels,omitempty"`
43 Comments []bundleComment `json:"comments,omitempty"`
44}
45type bundleMR struct {
46 Number int64 `json:"number"`
47 Title string `json:"title"`
48 Body string `json:"body"`
49 State string `json:"state"`
50 Author string `json:"author"`
51 SourceRef string `json:"source_ref"`
52 TargetRef string `json:"target_ref"`
53 // HeadSHA and MergedBase are what a diff is measured between. The
54 // bundle carried neither, so every migrated merge request arrived
55 // with an empty head and `mr diff` could only fail on it (#128).
56 // They are recorded whether or not the objects have been pushed yet;
57 // the ref is pointed at the head once they have.
58 HeadSHA string `json:"head_sha,omitempty"`
59 MergedBase string `json:"merged_base,omitempty"`
60 MergedAt string `json:"merged_at,omitempty"`
61 CreatedAt string `json:"created_at"`
62 Comments []bundleComment `json:"comments,omitempty"`
63}
64type bundleRepo struct {
65 Name string `json:"name"`
66 Visibility string `json:"visibility"`
67 DefaultBranch string `json:"default_branch"`
68 Description string `json:"description,omitempty"`
69 Topics []string `json:"topics,omitempty"`
70 Settings store.RepoSettings `json:"settings"`
71 Issues []bundleIssue `json:"issues,omitempty"`
72 MRs []bundleMR `json:"mrs,omitempty"`
73}
74type bundle struct {
75 Bundle string `json:"bundle"`
76 Username string `json:"username"`
77 Profile store.Profile `json:"profile"`
78 Emails []string `json:"emails,omitempty"`
79 Repos []bundleRepo `json:"repos"`
80}
81
82func runAccountExport(c *Ctx, args []string) int {
83 if len(args) != 0 {
84 return c.fail(protocol.ExitUsage, "usage: account export > bundle.json")
85 }
86 b := bundle{Bundle: bundleVersion, Username: c.User.Username}
87 b.Profile, _ = c.Store.OwnerProfile("user", c.User.ID)
88 b.Emails, _ = c.Store.UserEmailAddresses(c.User.ID)
89 repos, err := c.Store.ListReposForOwner("user", c.User.ID)
90 if err != nil {
91 return c.fail(protocol.ExitFailure, "%v", err)
92 }
93 for _, r := range repos {
94 br := bundleRepo{
95 Name: r.Name,
96 Visibility: r.Visibility,
97 DefaultBranch: r.DefaultBranch,
98 Description: gitutil.ReadDescription(RepoDir(c.Cfg.Server.Root, r.OwnerName, r.Name)),
99 Settings: r.Settings,
100 }
101 br.Topics, _ = c.Store.ListTopics(r.ID)
102 issues, _ := c.Store.ListIssues(r.ID, "all", 0, 0)
103 for i := len(issues) - 1; i >= 0; i-- { // ascending numbers
104 iss := issues[i]
105 full, err := c.Store.IssueByNumber(r.ID, iss.Number)
106 if err != nil {
107 continue
108 }
109 bi := bundleIssue{Number: full.Number, Title: full.Title, Body: full.Body,
110 State: full.State, Author: full.Author, CreatedAt: full.CreatedAt, Labels: full.Labels}
111 if cs, err := c.Store.ListIssueComments(full.ID); err == nil {
112 for _, cm := range cs {
113 bi.Comments = append(bi.Comments, bundleComment{cm.Author, cm.Body, cm.CreatedAt})
114 }
115 }
116 br.Issues = append(br.Issues, bi)
117 }
118 mrs, _ := c.Store.ListMRs(r.ID, "all", 0, 0)
119 for i := len(mrs) - 1; i >= 0; i-- {
120 m := mrs[i]
121 bm := bundleMR{Number: m.Number, Title: m.Title, Body: m.Body, State: m.State,
122 Author: m.Author, SourceRef: m.SourceRef, TargetRef: m.TargetRef,
123 HeadSHA: m.HeadSHA, MergedBase: m.MergedBase, MergedAt: m.MergedAt,
124 CreatedAt: m.CreatedAt}
125 if cs, err := c.Store.ListMRComments(m.ID); err == nil {
126 for _, cm := range cs {
127 bm.Comments = append(bm.Comments, bundleComment{cm.Author, cm.Body, cm.CreatedAt})
128 }
129 }
130 br.MRs = append(br.MRs, bm)
131 }
132 b.Repos = append(b.Repos, br)
133 }
134 enc := json.NewEncoder(c.Stdout)
135 enc.SetIndent("", " ")
136 if err := enc.Encode(b); err != nil {
137 return protocol.ExitFailure
138 }
139 return protocol.ExitOK
140}
141
142func migAttribution(src, kind, author, date string, n int64) string {
143 return fmt.Sprintf("> migrated %s %s#%d — %s, %.10s\n\n", kind, src, n, author, date)
144}
145
146// runAccountImportBundle replays a bundle under the calling user. Repos are
147// created empty (the CLI pushes git data with the user's own key); issues,
148// MRs, and comments arrive attributed inline. Push-blocking policies
149// (require_signed_commits, protected_branches) are deferred and reported so
150// the git push that follows cannot be refused by them. Resumable: markers
151// skip everything already imported.
152func runAccountImportBundle(c *Ctx, args []string) int {
153 f, err := parseFlags(args, flagSpec{Values: []string{"--source"}, MaxPos: 0, Usage: "account import-bundle [--source <host>] < bundle.json"})
154 if err != nil {
155 return c.fail(protocol.ExitUsage, "%v", err)
156 }
157 src := f.Value("--source")
158 if src == "" {
159 src = "the previous instance"
160 }
161 var b bundle
162 if err := json.NewDecoder(io.LimitReader(c.Stdin, 512<<20)).Decode(&b); err != nil {
163 return c.fail(protocol.ExitUsage, "bundle does not parse: %v", err)
164 }
165 if b.Bundle != bundleVersion {
166 return c.fail(protocol.ExitUsage, "unsupported bundle %q (want %s)", b.Bundle, bundleVersion)
167 }
168
169 if b.Profile.Description != "" || b.Profile.Website != "" ||
170 b.Profile.About != "" || len(b.Profile.Links) > 0 {
171 c.Store.SetOwnerProfile("user", c.User.ID, b.Profile)
172 }
173 for _, addr := range b.Emails {
174 c.Store.AddEmail(c.User.ID, addr, "", false) // unverified; re-verify here
175 }
176
177 type deferred struct {
178 Repo string `json:"repo"`
179 Commands []string `json:"commands"`
180 }
181 var repos, issues, mrs, comments, skipped int
182 var deferrals []deferred
183 for _, br := range b.Repos {
184 path := c.User.Username + "/" + br.Name
185 repo, err := c.Store.RepoByPath(path)
186 if err != nil {
187 id, err := c.Store.CreateRepo("user", c.User.ID, br.Name, br.Visibility)
188 if err != nil {
189 return c.fail(protocol.ExitFailure, "creating %s: %v", path, err)
190 }
191 dir := RepoDir(c.Cfg.Server.Root, c.User.Username, br.Name)
192 branch := br.DefaultBranch
193 if branch == "" {
194 branch = "main"
195 }
196 if err := gitutil.InitBare(dir, branch, HooksDir(c.Cfg.Server.Root)); err != nil {
197 c.Store.DeleteRepo(id)
198 return c.fail(protocol.ExitFailure, "initializing %s: %v", path, err)
199 }
200 if br.Description != "" {
201 gitutil.WriteDescription(dir, br.Description)
202 }
203 repo, err = c.Store.RepoByPath(path)
204 if err != nil {
205 return c.fail(protocol.ExitFailure, "%v", err)
206 }
207 repos++
208 }
209 for _, tpc := range br.Topics {
210 c.Store.AddTopic(repo.ID, tpc)
211 }
212 // Settings minus the two that would refuse the git push coming
213 // right after this; the user re-applies them once data is in.
214 s := br.Settings
215 var cmds []string
216 if s.RequireSignedCommits {
217 cmds = append(cmds, fmt.Sprintf("gitbay repo settings require-signed %s on", path))
218 s.RequireSignedCommits = false
219 }
220 if len(s.ProtectedBranches) > 0 {
221 for _, pb := range s.ProtectedBranches {
222 cmds = append(cmds, fmt.Sprintf("gitbay repo settings protect %s %s", path, pb))
223 }
224 s.ProtectedBranches = nil
225 }
226 if len(cmds) > 0 {
227 deferrals = append(deferrals, deferred{path, cmds})
228 }
229 // An import writes the whole blob: the repository was created a
230 // few lines up and nobody else holds settings on it yet.
231 s.GitDaemon = false // instance-dependent; opt back in explicitly
232 c.Store.UpdateRepoSettings(repo.ID, func(cur *store.RepoSettings) { *cur = s })
233
234 for _, bi := range br.Issues {
235 key := fmt.Sprintf("mig-issue:%d", bi.Number)
236 if _, seen, _ := c.Store.ImportMarker(repo.ID, key); seen {
237 skipped++
238 continue
239 }
240 body := migAttribution(src, "issue", bi.Author, bi.CreatedAt, bi.Number) + bi.Body
241 n, err := c.Store.CreateIssue(repo.ID, c.User.ID, bi.Title, body, "md")
242 if err != nil {
243 return c.fail(protocol.ExitFailure, "%v", err)
244 }
245 iss, err := c.Store.IssueByNumber(repo.ID, n)
246 if err != nil {
247 return c.fail(protocol.ExitFailure, "%v", err)
248 }
249 for _, l := range bi.Labels {
250 c.Store.SetIssueLabel(repo, iss.ID, l, true)
251 }
252 if bi.State != "open" {
253 c.Store.SetIssueState(iss.ID, "closed")
254 }
255 for _, cm := range bi.Comments {
256 c.Store.AddIssueComment(iss.ID, c.User.ID,
257 fmt.Sprintf("> %s, %.10s\n\n%s", cm.Author, cm.CreatedAt, cm.Body), "md")
258 comments++
259 }
260 c.Store.SetImportMarker(repo.ID, key, fmt.Sprint(n))
261 issues++
262 }
263 for _, bm := range br.MRs {
264 key := fmt.Sprintf("mig-mr:%d", bm.Number)
265 if val, seen, _ := c.Store.ImportMarker(repo.ID, key); seen {
266 skipped++
267 // A bundle is imported before the git push as often as
268 // after, so the objects a merge request needs may only
269 // have arrived since. Re-running the import is how the
270 // head ref gets set, and doing that costs nothing when it
271 // is already right.
272 if local, err := strconv.ParseInt(val, 10, 64); err == nil {
273 setMigratedHead(c, repo, local, bm.HeadSHA)
274 }
275 continue
276 }
277 body := migAttribution(src, "merge request", bm.Author, bm.CreatedAt, bm.Number) + bm.Body
278 n, err := c.Store.CreateMR(repo.ID, c.User.ID, repo.ID, bm.SourceRef, bm.TargetRef, bm.Title, body, bm.HeadSHA, "md", false)
279 if err != nil {
280 return c.fail(protocol.ExitFailure, "%v", err)
281 }
282 mr, err := c.Store.MRByNumber(repo.ID, n)
283 if err != nil {
284 return c.fail(protocol.ExitFailure, "%v", err)
285 }
286 switch {
287 case bm.State == "merged":
288 // MarkMerged, not SetMRState: a merged MR's diff is
289 // measured from the base recorded at merge time, and
290 // SetMRState leaves that empty.
291 c.Store.MarkMerged(mr.ID, bm.MergedBase, 0, bm.MergedAt)
292 case bm.State != "open":
293 state := bm.State
294 if state == "source_gone" {
295 state = "closed"
296 }
297 c.Store.SetMRState(mr.ID, state)
298 }
299 setMigratedHead(c, repo, n, bm.HeadSHA)
300 for _, cm := range bm.Comments {
301 c.Store.AddMRComment(mr.ID, c.User.ID,
302 fmt.Sprintf("> %s, %.10s\n\n%s", cm.Author, cm.CreatedAt, cm.Body), "md")
303 comments++
304 }
305 c.Store.SetImportMarker(repo.ID, key, fmt.Sprint(n))
306 mrs++
307 }
308 }
309 d := map[string]any{
310 "repos": repos, "issues": issues, "mrs": mrs, "comments": comments,
311 "already_imported": skipped, "deferred_settings": deferrals,
312 }
313 return c.emit(d, func(w io.Writer) {
314 fmt.Fprintf(w, "imported %d repos, %d issues, %d MRs, %d comments (%d already present)\n",
315 repos, issues, mrs, comments, skipped)
316 if len(deferrals) > 0 {
317 fmt.Fprintf(w, "\nafter pushing git data, re-apply the deferred policies:\n")
318 for _, df := range deferrals {
319 for _, cmd := range df.Commands {
320 fmt.Fprintf(w, " %s\n", cmd)
321 }
322 }
323 }
324 })
325}
326
327var _ = strings.TrimSpace // placeholder against accidental import drops
328
329// setMigratedHead points refs/merge-requests/<n>/head at the head a
330// bundle recorded, once the objects for it are present. `mr diff`
331// resolves through that ref, not through the stored head_sha, so a merge
332// request whose source branch is gone — every merged one — has nothing to
333// diff without it (#128).
334//
335// Best-effort by design: a bundle is imported before the git push as
336// often as after. head_sha is stored either way, and running the import
337// again after the push sets the ref.
338func setMigratedHead(c *Ctx, repo store.Repo, number int64, headSHA string) {
339 if headSHA == "" {
340 return
341 }
342 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
343 if !gitutil.HasCommit(dir, headSHA) {
344 return
345 }
346 ref := fmt.Sprintf("refs/merge-requests/%d/head", number)
347 if cur, err := gitutil.ResolveRef(dir, ref); err == nil && cur == headSHA {
348 return
349 }
350 gitutil.UpdateRefCAS(dir, ref, headSHA, "")
351}