internal/control/ghimport.go
290 lines · 9108 bytes
1package control
2
3import (
4 "bufio"
5 "encoding/json"
6 "fmt"
7 "io"
8 "net/http"
9 "net/url"
10 "strconv"
11 "strings"
12 "time"
13
14 "gitbay.org/gitbay/internal/gitutil"
15 "gitbay.org/gitbay/internal/policy"
16 "gitbay.org/gitbay/internal/protocol"
17 "gitbay.org/gitbay/internal/store"
18 "gitbay.org/gitbay/internal/webhook"
19)
20
21func init() {
22 register(Command{Path: []string{"repo", "import-issues"},
23 Summary: "import GitHub issue and PR history: repo import-issues <owner/name> --from <ghowner/ghrepo> [--token-stdin] [--api-base <url>]",
24 ReadsStdin: true, Run: runImportIssues})
25}
26
27// GitHub API shapes, minimal.
28type ghUser struct {
29 Login string `json:"login"`
30}
31type ghIssue struct {
32 Number int64 `json:"number"`
33 Title string `json:"title"`
34 Body string `json:"body"`
35 State string `json:"state"`
36 CreatedAt string `json:"created_at"`
37 User ghUser `json:"user"`
38 Labels []struct{ Name string } `json:"labels"`
39 PullRequest *struct{} `json:"pull_request"`
40 Comments int `json:"comments"`
41}
42type ghPull struct {
43 MergedAt string `json:"merged_at"`
44 Head struct {
45 SHA string `json:"sha"`
46 Ref string `json:"ref"`
47 } `json:"head"`
48 Base struct {
49 SHA string `json:"sha"`
50 Ref string `json:"ref"`
51 } `json:"base"`
52}
53type ghComment struct {
54 ID int64 `json:"id"`
55 Body string `json:"body"`
56 CreatedAt string `json:"created_at"`
57 User ghUser `json:"user"`
58}
59
60type ghClient struct {
61 base string
62 token string
63 http *http.Client
64}
65
66func (g *ghClient) get(path string, out any) error {
67 req, err := http.NewRequest("GET", g.base+path, nil)
68 if err != nil {
69 return err
70 }
71 req.Header.Set("Accept", "application/vnd.github+json")
72 if g.token != "" {
73 req.Header.Set("Authorization", "Bearer "+g.token)
74 }
75 resp, err := g.http.Do(req)
76 if err != nil {
77 return err
78 }
79 defer resp.Body.Close()
80 if resp.StatusCode != 200 {
81 body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
82 return fmt.Errorf("GitHub API %s: %s: %.200s", path, resp.Status, body)
83 }
84 return json.NewDecoder(resp.Body).Decode(out)
85}
86
87func ghDate(iso string) string {
88 if t, err := time.Parse(time.RFC3339, iso); err == nil {
89 return t.UTC().Format("2006-01-02")
90 }
91 return iso
92}
93
94// attribution heads every imported body: foreign authors have no local
95// account, so the original author and date live in the text.
96func attribution(src string, n int64, kind, login, date string) string {
97 return fmt.Sprintf("> imported %s %s#%d — @%s, %s\n\n", kind, src, n, login, ghDate(date))
98}
99
100func runImportIssues(c *Ctx, args []string) int {
101 var path, from, apiBase string
102 tokenStdin := false
103 for i := 0; i < len(args); i++ {
104 switch args[i] {
105 case "--from", "--api-base":
106 if i+1 >= len(args) {
107 return c.fail(protocol.ExitUsage, "%s requires a value", args[i])
108 }
109 if args[i] == "--from" {
110 from = args[i+1]
111 } else {
112 apiBase = args[i+1]
113 }
114 i++
115 case "--token-stdin":
116 tokenStdin = true
117 default:
118 if path != "" {
119 return c.fail(protocol.ExitUsage, "unexpected argument %q", args[i])
120 }
121 path = args[i]
122 }
123 }
124 if path == "" || from == "" {
125 return c.fail(protocol.ExitUsage, "usage: repo import-issues <owner/name> --from <ghowner/ghrepo> [--token-stdin]")
126 }
127 // Accept a bare owner/repo or a full github.com URL.
128 from = strings.TrimSuffix(strings.TrimPrefix(strings.TrimPrefix(from, "https://"), "github.com/"), ".git")
129 if parts := strings.Split(from, "/"); len(parts) != 2 || parts[0] == "" || parts[1] == "" {
130 return c.fail(protocol.ExitUsage, "--from must be <ghowner>/<ghrepo> (or the github.com URL)")
131 }
132 if apiBase == "" {
133 apiBase = "https://api.github.com"
134 } else if err := webhook.ValidateURL(apiBase, c.Cfg.Webhooks.AllowLocal); err != nil {
135 // A writer-supplied API base is the same SSRF surface as a
136 // webhook target; same rules apply.
137 return c.fail(protocol.ExitUsage, "--api-base: %v", err)
138 }
139 repo, code := resolveRepo(c, path, policy.CanWrite)
140 if code >= 0 {
141 return code
142 }
143 if code := refuseArchived(c, repo); code >= 0 {
144 return code
145 }
146 token := ""
147 if tokenStdin {
148 // Same discipline as repo import: token on stdin, never argv,
149 // never stored.
150 line, err := bufio.NewReader(c.Stdin).ReadString('\n')
151 if err != nil && line == "" {
152 return c.fail(protocol.ExitUsage, "--token-stdin: no token on stdin")
153 }
154 token = strings.TrimSpace(line)
155 }
156 g := &ghClient{base: apiBase, token: token, http: &http.Client{Timeout: 30 * time.Second}}
157 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
158 src := "github.com/" + from
159
160 var issues, mrs, comments, skipped int
161 for page := 1; ; page++ {
162 var items []ghIssue
163 q := fmt.Sprintf("/repos/%s/issues?state=all&sort=created&direction=asc&per_page=100&page=%d", from, page)
164 if err := g.get(q, &items); err != nil {
165 return c.fail(protocol.ExitFailure, "%v", err)
166 }
167 if len(items) == 0 {
168 break
169 }
170 for _, it := range items {
171 key := fmt.Sprintf("gh:%d", it.Number)
172 val, seen, err := c.Store.ImportMarker(repo.ID, key)
173 if err != nil {
174 return c.fail(protocol.ExitFailure, "%v", err)
175 }
176 var localN int64
177 isPR := it.PullRequest != nil
178 if seen {
179 localN, _ = strconv.ParseInt(strings.TrimPrefix(strings.TrimPrefix(val, "issue:"), "mr:"), 10, 64)
180 skipped++
181 } else if isPR {
182 var pr ghPull
183 if err := g.get(fmt.Sprintf("/repos/%s/pulls/%d", from, it.Number), &pr); err != nil {
184 return c.fail(protocol.ExitFailure, "%v", err)
185 }
186 body := attribution(src, it.Number, "pull request", it.User.Login, it.CreatedAt) + it.Body
187 localN, err = c.Store.CreateMR(repo.ID, c.User.ID, repo.ID, pr.Head.Ref, pr.Base.Ref, it.Title, body, pr.Head.SHA, "md")
188 if err != nil {
189 return c.fail(protocol.ExitFailure, "%v", err)
190 }
191 mr, err := c.Store.MRByNumber(repo.ID, localN)
192 if err != nil {
193 return c.fail(protocol.ExitFailure, "%v", err)
194 }
195 if pr.MergedAt != "" {
196 c.Store.MarkMerged(mr.ID, pr.Base.SHA)
197 } else {
198 c.Store.SetMRState(mr.ID, "closed")
199 }
200 // Point the MR head ref at the PR head when the mirror
201 // already holds the objects (refs/pull backups).
202 if pr.Head.SHA != "" && gitutil.HasCommit(dir, pr.Head.SHA) {
203 gitutil.UpdateRefCAS(dir, fmt.Sprintf("refs/merge-requests/%d/head", localN), pr.Head.SHA, "")
204 }
205 c.Store.SetImportMarker(repo.ID, key, fmt.Sprintf("mr:%d", localN))
206 mrs++
207 } else {
208 body := attribution(src, it.Number, "issue", it.User.Login, it.CreatedAt) + it.Body
209 localN, err = c.Store.CreateIssue(repo.ID, c.User.ID, it.Title, body, "md")
210 if err != nil {
211 return c.fail(protocol.ExitFailure, "%v", err)
212 }
213 iss, err := c.Store.IssueByNumber(repo.ID, localN)
214 if err != nil {
215 return c.fail(protocol.ExitFailure, "%v", err)
216 }
217 for _, l := range it.Labels {
218 c.Store.SetIssueLabel(repo.ID, iss.ID, l.Name, true)
219 }
220 if it.State != "open" {
221 c.Store.SetIssueState(iss.ID, "closed")
222 }
223 c.Store.SetImportMarker(repo.ID, key, fmt.Sprintf("issue:%d", localN))
224 issues++
225 }
226 if it.Comments > 0 && localN > 0 {
227 n, err := importComments(c, g, repo, from, src, it.Number, localN, isPR)
228 if err != nil {
229 return c.fail(protocol.ExitFailure, "%v", err)
230 }
231 comments += n
232 }
233 fmt.Fprintf(c.Stderr, "%s#%d -> %s%d\n", src, it.Number, map[bool]string{true: "!", false: "#"}[isPR], localN)
234 }
235 }
236 d := map[string]any{"issues": issues, "mrs": mrs, "comments": comments, "already_imported": skipped}
237 return c.emit(d, func(w io.Writer) {
238 fmt.Fprintf(w, "imported %d issues, %d merge requests, %d comments (%d items already imported)\n",
239 issues, mrs, comments, skipped)
240 })
241}
242
243func importComments(c *Ctx, g *ghClient, repo store.Repo, from, src string, ghN, localN int64, isPR bool) (int, error) {
244 var localIssueID, localMRID int64
245 if isPR {
246 mr, err := c.Store.MRByNumber(repo.ID, localN)
247 if err != nil {
248 return 0, err
249 }
250 localMRID = mr.ID
251 } else {
252 iss, err := c.Store.IssueByNumber(repo.ID, localN)
253 if err != nil {
254 return 0, err
255 }
256 localIssueID = iss.ID
257 }
258 imported := 0
259 for page := 1; ; page++ {
260 var cs []ghComment
261 q := fmt.Sprintf("/repos/%s/issues/%d/comments?per_page=100&page=%d", url.PathEscape(from), ghN, page)
262 q = strings.ReplaceAll(q, "%2F", "/")
263 if err := g.get(q, &cs); err != nil {
264 return imported, err
265 }
266 if len(cs) == 0 {
267 return imported, nil
268 }
269 for _, cm := range cs {
270 key := fmt.Sprintf("ghc:%d", cm.ID)
271 if _, seen, err := c.Store.ImportMarker(repo.ID, key); err != nil {
272 return imported, err
273 } else if seen {
274 continue
275 }
276 body := fmt.Sprintf("> @%s, %s\n\n%s", cm.User.Login, ghDate(cm.CreatedAt), cm.Body)
277 var err error
278 if isPR {
279 err = c.Store.AddMRComment(localMRID, c.User.ID, body, "md")
280 } else {
281 err = c.Store.AddIssueComment(localIssueID, c.User.ID, body, "md")
282 }
283 if err != nil {
284 return imported, err
285 }
286 c.Store.SetImportMarker(repo.ID, key, "")
287 imported++
288 }
289 }
290}