internal/control/ghimport.go
373 lines · 12908 bytes
1package control
2
3import (
4 "bufio"
5 "context"
6 "encoding/json"
7 "fmt"
8 "io"
9 "net/http"
10 "net/url"
11 "os"
12 "path/filepath"
13 "strconv"
14 "strings"
15 "time"
16
17 "gitbay.org/gitbay/internal/gitutil"
18 "gitbay.org/gitbay/internal/policy"
19 "gitbay.org/gitbay/internal/protocol"
20 "gitbay.org/gitbay/internal/store"
21 "gitbay.org/gitbay/internal/webhook"
22)
23
24func init() {
25 register(Command{Path: []string{"repo", "import-issues"},
26 Summary: "import issue and PR history from GitHub or Forgejo",
27 Usage: "repo import-issues <owner/name> --from <owner/repo> [--token-stdin] [--api-base <url>]",
28 ReadsStdin: true, Run: runImportIssues})
29}
30
31// GitHub API shapes, minimal.
32type ghUser struct {
33 Login string `json:"login"`
34}
35type ghIssue struct {
36 Number int64 `json:"number"`
37 Title string `json:"title"`
38 Body string `json:"body"`
39 State string `json:"state"`
40 CreatedAt string `json:"created_at"`
41 ClosedAt string `json:"closed_at"`
42 User ghUser `json:"user"`
43 Labels []struct{ Name string } `json:"labels"`
44 PullRequest *struct{} `json:"pull_request"`
45 Comments int `json:"comments"`
46}
47type ghPull struct {
48 MergedAt string `json:"merged_at"`
49 Head struct {
50 SHA string `json:"sha"`
51 Ref string `json:"ref"`
52 } `json:"head"`
53 Base struct {
54 SHA string `json:"sha"`
55 Ref string `json:"ref"`
56 } `json:"base"`
57}
58type ghComment struct {
59 ID int64 `json:"id"`
60 Body string `json:"body"`
61 CreatedAt string `json:"created_at"`
62 User ghUser `json:"user"`
63}
64
65type ghClient struct {
66 base string
67 token string
68 http *http.Client
69 // forgejo is set when the API base answers /version, which GitHub
70 // does not. Forgejo (and Gitea, and so Codeberg) mirror GitHub's
71 // issue, pull and comment shapes but not its query parameters:
72 // pages are sized by `limit`, order is `sort=oldest`, and the
73 // comments endpoint has no pages at all — it ignores `page` and
74 // returns everything every time, which paged the old loop forever.
75 forgejo bool
76}
77
78func (g *ghClient) detect() {
79 var v struct{ Version string }
80 g.forgejo = g.get("/version", &v) == nil && v.Version != ""
81}
82
83// issuesQuery lists every issue and pull request oldest first, so local
84// numbers come out in the source's order.
85func (g *ghClient) issuesQuery(from string, page int) string {
86 if g.forgejo {
87 return fmt.Sprintf("/repos/%s/issues?state=all&sort=oldest&limit=50&page=%d", from, page)
88 }
89 return fmt.Sprintf("/repos/%s/issues?state=all&sort=created&direction=asc&per_page=100&page=%d", from, page)
90}
91
92// siteFromAPI turns an API base into the site that serves git and the
93// name attribution carries: api.github.com is github.com, a GitHub
94// Enterprise or Forgejo base drops its /api/vN suffix.
95func siteFromAPI(apiBase string) string {
96 u, err := url.Parse(apiBase)
97 if err != nil {
98 return apiBase
99 }
100 if u.Host == "api.github.com" {
101 u.Host = "github.com"
102 u.Path = ""
103 }
104 u.Path = strings.TrimSuffix(strings.TrimSuffix(u.Path, "/"), "/api/v1")
105 u.Path = strings.TrimSuffix(u.Path, "/api/v3")
106 u.RawQuery, u.Fragment = "", ""
107 return u.String()
108}
109
110func (g *ghClient) get(path string, out any) error {
111 req, err := http.NewRequest("GET", g.base+path, nil)
112 if err != nil {
113 return err
114 }
115 req.Header.Set("Accept", "application/vnd.github+json")
116 if g.token != "" {
117 req.Header.Set("Authorization", "Bearer "+g.token)
118 }
119 resp, err := g.http.Do(req)
120 if err != nil {
121 return err
122 }
123 defer resp.Body.Close()
124 if resp.StatusCode != 200 {
125 body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
126 return fmt.Errorf("GitHub API %s: %s: %.200s", path, resp.Status, body)
127 }
128 return json.NewDecoder(resp.Body).Decode(out)
129}
130
131func ghDate(iso string) string {
132 if t, err := time.Parse(time.RFC3339, iso); err == nil {
133 return t.UTC().Format("2006-01-02")
134 }
135 return iso
136}
137
138// attribution heads every imported body: foreign authors have no local
139// account, so the original author and date live in the text.
140func attribution(src string, n int64, kind, login, date string) string {
141 return fmt.Sprintf("> imported %s %s#%d — @%s, %s\n\n", kind, src, n, login, ghDate(date))
142}
143
144func runImportIssues(c *Ctx, args []string) int {
145 f, err := parseFlags(args, flagSpec{Values: []string{"--from", "--api-base"}, Bools: []string{"--token-stdin"}, MaxPos: 1,
146 Usage: "repo import-issues <owner/name> --from <owner/repo> [--api-base <url>] [--token-stdin]"})
147 if err != nil {
148 return c.fail(protocol.ExitUsage, "%v", err)
149 }
150 path, from, apiBase, tokenStdin := f.pos(0), f.Value("--from"), f.Value("--api-base"), f.Has("--token-stdin")
151 if path == "" || from == "" {
152 return c.fail(protocol.ExitUsage, "usage: repo import-issues <owner/name> --from <owner/repo> [--token-stdin] [--api-base <url>]")
153 }
154 if apiBase == "" {
155 apiBase = "https://api.github.com"
156 } else if err := webhook.ValidateURL(apiBase, c.Cfg.Webhooks.AllowLocal); err != nil {
157 // A writer-supplied API base is the same SSRF surface as a
158 // webhook target; same rules apply.
159 return c.fail(protocol.ExitUsage, "--api-base: %v", err)
160 }
161 site := siteFromAPI(apiBase)
162 host := strings.TrimPrefix(strings.TrimPrefix(site, "https://"), "http://")
163 // Accept a bare owner/repo or the repository's URL on that site.
164 from = strings.TrimPrefix(from, site+"/")
165 from = strings.TrimPrefix(from, host+"/")
166 from = strings.TrimSuffix(from, ".git")
167 if parts := strings.Split(from, "/"); len(parts) != 2 || parts[0] == "" || parts[1] == "" {
168 return c.fail(protocol.ExitUsage, "--from must be <owner>/<repo> (or the repository URL on %s)", site)
169 }
170 repo, code := resolveRepo(c, path, policy.CanWrite)
171 if code >= 0 {
172 return code
173 }
174 if code := refuseArchived(c, repo); code >= 0 {
175 return code
176 }
177 token := ""
178 if tokenStdin {
179 // Same discipline as repo import: token on stdin, never argv,
180 // never stored.
181 line, err := bufio.NewReader(c.Stdin).ReadString('\n')
182 if err != nil && line == "" {
183 return c.fail(protocol.ExitUsage, "--token-stdin: no token on stdin")
184 }
185 token = strings.TrimSpace(line)
186 }
187 g := &ghClient{base: apiBase, token: token, http: &http.Client{Timeout: 30 * time.Second}}
188 g.detect()
189 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
190
191 // Pull heads first, so every merge request below has objects to point
192 // at. A mirror made with the default refspecs does not carry
193 // refs/pull/*, which is why imported pull requests used to have no
194 // head and `mr diff` could only fail on them (#128). Best-effort: an
195 // import of issues from a repository whose git data is not here yet
196 // is a legitimate thing to do, and the merge requests still arrive
197 // with their head SHA recorded.
198 fetchedPullHeads := fetchPullHeads(c, dir, site+"/"+from+".git", token)
199 src := host + "/" + from
200
201 var issues, mrs, comments, skipped, headed int
202 for page := 1; ; page++ {
203 var items []ghIssue
204 if err := g.get(g.issuesQuery(from, page), &items); err != nil {
205 return c.fail(protocol.ExitFailure, "%v", err)
206 }
207 if len(items) == 0 {
208 break
209 }
210 for _, it := range items {
211 key := fmt.Sprintf("gh:%d", it.Number)
212 val, seen, err := c.Store.ImportMarker(repo.ID, key)
213 if err != nil {
214 return c.fail(protocol.ExitFailure, "%v", err)
215 }
216 var localN int64
217 isPR := it.PullRequest != nil
218 if seen {
219 localN, _ = strconv.ParseInt(strings.TrimPrefix(strings.TrimPrefix(val, "issue:"), "mr:"), 10, 64)
220 skipped++
221 } else if isPR {
222 var pr ghPull
223 if err := g.get(fmt.Sprintf("/repos/%s/pulls/%d", from, it.Number), &pr); err != nil {
224 return c.fail(protocol.ExitFailure, "%v", err)
225 }
226 body := attribution(src, it.Number, "pull request", it.User.Login, it.CreatedAt) + it.Body
227 localN, err = c.Store.CreateMR(repo.ID, c.User.ID, repo.ID, pr.Head.Ref, pr.Base.Ref, it.Title, body, pr.Head.SHA, "md", false)
228 if err != nil {
229 return c.fail(protocol.ExitFailure, "%v", err)
230 }
231 mr, err := c.Store.MRByNumber(repo.ID, localN)
232 if err != nil {
233 return c.fail(protocol.ExitFailure, "%v", err)
234 }
235 if pr.MergedAt != "" {
236 c.Store.MarkMerged(mr.ID, pr.Base.SHA, 0, pr.MergedAt)
237 } else {
238 c.Store.MarkClosed(mr.ID, 0, it.ClosedAt)
239 }
240 // Point the MR head ref at the PR head, from the fetch
241 // above or from objects a mirror already had.
242 if pr.Head.SHA != "" && gitutil.HasCommit(dir, pr.Head.SHA) {
243 gitutil.UpdateRefCAS(dir, fmt.Sprintf("refs/merge-requests/%d/head", localN), pr.Head.SHA, "")
244 headed++
245 }
246 c.Store.SetImportMarker(repo.ID, key, fmt.Sprintf("mr:%d", localN))
247 mrs++
248 } else {
249 body := attribution(src, it.Number, "issue", it.User.Login, it.CreatedAt) + it.Body
250 localN, err = c.Store.CreateIssue(repo.ID, c.User.ID, it.Title, body, "md")
251 if err != nil {
252 return c.fail(protocol.ExitFailure, "%v", err)
253 }
254 iss, err := c.Store.IssueByNumber(repo.ID, localN)
255 if err != nil {
256 return c.fail(protocol.ExitFailure, "%v", err)
257 }
258 for _, l := range it.Labels {
259 c.Store.SetIssueLabel(repo.ID, iss.ID, l.Name, true)
260 }
261 if it.State != "open" {
262 c.Store.SetIssueState(iss.ID, "closed")
263 }
264 c.Store.SetImportMarker(repo.ID, key, fmt.Sprintf("issue:%d", localN))
265 issues++
266 }
267 if it.Comments > 0 && localN > 0 {
268 n, err := importComments(c, g, repo, from, src, it.Number, localN, isPR)
269 if err != nil {
270 return c.fail(protocol.ExitFailure, "%v", err)
271 }
272 comments += n
273 }
274 fmt.Fprintf(c.Stderr, "%s#%d -> %s%d\n", src, it.Number, map[bool]string{true: "!", false: "#"}[isPR], localN)
275 }
276 }
277 d := map[string]any{"issues": issues, "mrs": mrs, "comments": comments,
278 "already_imported": skipped, "mrs_with_head": headed, "pull_heads_fetched": fetchedPullHeads}
279 return c.emit(d, func(w io.Writer) {
280 fmt.Fprintf(w, "imported %d issues, %d merge requests, %d comments (%d items already imported)\n",
281 issues, mrs, comments, skipped)
282 if mrs > headed {
283 fmt.Fprintf(w, "%d merge request(s) have no head objects; `mr diff` cannot render them.\n", mrs-headed)
284 if !fetchedPullHeads {
285 fmt.Fprintln(w, "refs/pull/* could not be fetched — re-run with --token-stdin if the repository is private.")
286 }
287 }
288 })
289}
290
291func importComments(c *Ctx, g *ghClient, repo store.Repo, from, src string, ghN, localN int64, isPR bool) (int, error) {
292 var localIssueID, localMRID int64
293 if isPR {
294 mr, err := c.Store.MRByNumber(repo.ID, localN)
295 if err != nil {
296 return 0, err
297 }
298 localMRID = mr.ID
299 } else {
300 iss, err := c.Store.IssueByNumber(repo.ID, localN)
301 if err != nil {
302 return 0, err
303 }
304 localIssueID = iss.ID
305 }
306 imported := 0
307 for page := 1; ; page++ {
308 // Forgejo returns every comment in one unpaged reply.
309 if g.forgejo && page > 1 {
310 return imported, nil
311 }
312 var cs []ghComment
313 q := fmt.Sprintf("/repos/%s/issues/%d/comments?per_page=100&page=%d", url.PathEscape(from), ghN, page)
314 q = strings.ReplaceAll(q, "%2F", "/")
315 if err := g.get(q, &cs); err != nil {
316 return imported, err
317 }
318 if len(cs) == 0 {
319 return imported, nil
320 }
321 for _, cm := range cs {
322 key := fmt.Sprintf("ghc:%d", cm.ID)
323 if _, seen, err := c.Store.ImportMarker(repo.ID, key); err != nil {
324 return imported, err
325 } else if seen {
326 continue
327 }
328 body := fmt.Sprintf("> @%s, %s\n\n%s", cm.User.Login, ghDate(cm.CreatedAt), cm.Body)
329 var err error
330 if isPR {
331 err = c.Store.AddMRComment(localMRID, c.User.ID, body, "md")
332 } else {
333 err = c.Store.AddIssueComment(localIssueID, c.User.ID, body, "md")
334 }
335 if err != nil {
336 return imported, err
337 }
338 c.Store.SetImportMarker(repo.ID, key, "")
339 imported++
340 }
341 }
342}
343
344// ghAskpass answers git's credential prompts from the environment, so a
345// token never appears in argv where /proc would expose it. Same shape the
346// mirror worker uses.
347const ghAskpass = `#!/bin/sh
348case "$1" in
349 Username*) echo "x-access-token" ;;
350 *) echo "${GITBAY_GH_TOKEN}" ;;
351esac
352`
353
354// fetchPullHeads brings refs/pull/*/head into refs/gh-pull/*; GitHub and
355// Forgejo both publish pull heads under that name. Reports whether it
356// worked; a failure is not fatal, since importing issues from a
357// repository whose git data is not here yet is a reasonable thing to do.
358func fetchPullHeads(c *Ctx, dir, remote, token string) bool {
359 env := []string{"GIT_TERMINAL_PROMPT=0", "HOME=" + c.Cfg.Server.Root}
360 if token != "" {
361 askpass := filepath.Join(c.Cfg.Server.Root, "gh-import-askpass.sh")
362 if err := os.WriteFile(askpass, []byte(ghAskpass), 0o700); err != nil {
363 return false
364 }
365 env = append(env, "GIT_ASKPASS="+askpass, "GITBAY_GH_TOKEN="+token)
366 }
367 ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
368 defer cancel()
369 if err := gitutil.FetchPullHeads(ctx, dir, remote, io.Discard, env); err != nil {
370 return false
371 }
372 return true
373}