internal/control/ghimport.go
417 lines · 14807 bytes
1package control
2
3import (
4 "bufio"
5 "context"
6 "encoding/json"
7 "fmt"
8 "io"
9 "net/http"
10 "net/url"
11 "os"
12 "path/filepath"
13 "strconv"
14 "strings"
15 "time"
16
17 "gitbay.org/gitbay/internal/gitpin"
18 "gitbay.org/gitbay/internal/gitutil"
19 "gitbay.org/gitbay/internal/policy"
20 "gitbay.org/gitbay/internal/protocol"
21 "gitbay.org/gitbay/internal/store"
22)
23
24func init() {
25 register(Command{Path: []string{"repo", "import-issues"},
26 Summary: "import issue and PR history from GitHub or Forgejo",
27 Usage: "repo import-issues <owner/name> --from <owner/repo> [--token-stdin] [--api-base <url>]",
28 Flags: []Flag{
29 {"--from", "<owner/repo>", "the source repository", ""},
30 {"--token-stdin", "", "read an API token from stdin", ""},
31 {"--api-base", "<url>", "the API's base URL, for Forgejo", ""},
32 },
33 Examples: []string{"repo import-issues krz/gitbay --from krz/gitbay-old"},
34 ReadsStdin: true, Run: runImportIssues})
35}
36
37// GitHub API shapes, minimal.
38type ghUser struct {
39 Login string `json:"login"`
40}
41type ghIssue struct {
42 Number int64 `json:"number"`
43 Title string `json:"title"`
44 Body string `json:"body"`
45 State string `json:"state"`
46 CreatedAt string `json:"created_at"`
47 ClosedAt string `json:"closed_at"`
48 User ghUser `json:"user"`
49 Labels []struct{ Name string } `json:"labels"`
50 PullRequest *struct{} `json:"pull_request"`
51 Comments int `json:"comments"`
52}
53type ghPull struct {
54 MergedAt string `json:"merged_at"`
55 Head struct {
56 SHA string `json:"sha"`
57 Ref string `json:"ref"`
58 } `json:"head"`
59 Base struct {
60 SHA string `json:"sha"`
61 Ref string `json:"ref"`
62 } `json:"base"`
63}
64type ghComment struct {
65 ID int64 `json:"id"`
66 Body string `json:"body"`
67 CreatedAt string `json:"created_at"`
68 User ghUser `json:"user"`
69}
70
71type ghClient struct {
72 base string
73 token string
74 http *http.Client
75 // forgejo is set when the API base answers /version, which GitHub
76 // does not. Forgejo (and Gitea, and so Codeberg) mirror GitHub's
77 // issue, pull and comment shapes but not its query parameters:
78 // pages are sized by `limit`, order is `sort=oldest`, and the
79 // comments endpoint has no pages at all — it ignores `page` and
80 // returns everything every time, which paged the old loop forever.
81 forgejo bool
82}
83
84func (g *ghClient) detect() {
85 var v struct{ Version string }
86 g.forgejo = g.get("/version", &v) == nil && v.Version != ""
87}
88
89// issuesQuery lists every issue and pull request oldest first, so local
90// numbers come out in the source's order.
91func (g *ghClient) issuesQuery(from string, page int) string {
92 if g.forgejo {
93 return fmt.Sprintf("/repos/%s/issues?state=all&sort=oldest&limit=50&page=%d", from, page)
94 }
95 return fmt.Sprintf("/repos/%s/issues?state=all&sort=created&direction=asc&per_page=100&page=%d", from, page)
96}
97
98// siteFromAPI turns an API base into the site that serves git and the
99// name attribution carries: api.github.com is github.com, a GitHub
100// Enterprise or Forgejo base drops its /api/vN suffix.
101func siteFromAPI(apiBase string) string {
102 u, err := url.Parse(apiBase)
103 if err != nil {
104 return apiBase
105 }
106 if u.Host == "api.github.com" {
107 u.Host = "github.com"
108 u.Path = ""
109 }
110 u.Path = strings.TrimSuffix(strings.TrimSuffix(u.Path, "/"), "/api/v1")
111 u.Path = strings.TrimSuffix(u.Path, "/api/v3")
112 u.RawQuery, u.Fragment = "", ""
113 return u.String()
114}
115
116func (g *ghClient) get(path string, out any) error {
117 req, err := http.NewRequest("GET", g.base+path, nil)
118 if err != nil {
119 return err
120 }
121 req.Header.Set("Accept", "application/vnd.github+json")
122 if g.token != "" {
123 req.Header.Set("Authorization", "Bearer "+g.token)
124 }
125 resp, err := g.http.Do(req)
126 if err != nil {
127 return err
128 }
129 defer resp.Body.Close()
130 if resp.StatusCode != 200 {
131 body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
132 return fmt.Errorf("GitHub API %s: %s: %.200s", path, resp.Status, body)
133 }
134 return json.NewDecoder(resp.Body).Decode(out)
135}
136
137// pinnedClient reaches api's host only at its checked addresses, never
138// through a proxy from the environment, and follows no redirect, as
139// webhook delivery and repo import do. Each import builds its own
140// client, so connections are not kept for reuse.
141func pinnedClient(api gitpin.Remote) *http.Client {
142 return &http.Client{
143 Timeout: 30 * time.Second,
144 Transport: &http.Transport{Proxy: nil, DialContext: api.DialContext, TLSHandshakeTimeout: 10 * time.Second, DisableKeepAlives: true},
145 CheckRedirect: func(req *http.Request, _ []*http.Request) error {
146 return fmt.Errorf("refusing redirect to %s://%s; a renamed repository is imported under its new name", req.URL.Scheme, req.URL.Host)
147 },
148 }
149}
150
151func ghDate(iso string) string {
152 if t, err := time.Parse(time.RFC3339, iso); err == nil {
153 return t.UTC().Format("2006-01-02")
154 }
155 return iso
156}
157
158// attribution heads every imported body: foreign authors have no local
159// account, so the original author and date live in the text.
160func attribution(src string, n int64, kind, login, date string) string {
161 return fmt.Sprintf("> imported %s %s#%d — @%s, %s\n\n", kind, src, n, login, ghDate(date))
162}
163
164func runImportIssues(c *Ctx, args []string) int {
165 f, err := c.parseArgs(args, flagSpec{Values: []string{"--from", "--api-base"}, Bools: []string{"--token-stdin"}, MaxPos: 1,
166 Usage: "repo import-issues <owner/name> --from <owner/repo> [--api-base <url>] [--token-stdin]"})
167 if err != nil {
168 return c.fail(protocol.ExitUsage, "%v", err)
169 }
170 path, from, apiBase, tokenStdin := f.pos(0), f.Value("--from"), f.Value("--api-base"), f.Has("--token-stdin")
171 if path == "" || from == "" {
172 return c.usage()
173 }
174 given := apiBase != ""
175 if !given {
176 apiBase = "https://api.github.com"
177 } else if strings.Contains(apiBase, "@") || strings.ContainsAny(apiBase, "?#") {
178 // Same rule as repo import: the URL is echoed and becomes the
179 // site prefix, so credentials and queries stay out of it.
180 return c.fail(protocol.ExitUsage, "--api-base: use the plain API URL, without credentials, a query or a fragment; a token goes on stdin with --token-stdin")
181 }
182 // A writer-supplied API base is the same SSRF surface as a webhook
183 // target. Resolve and check it once here; the client connects only
184 // to those addresses (#301).
185 rctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
186 api, err := gitpin.Resolve(rctx, importLookup, apiBase, c.Cfg.Webhooks.AllowLocal)
187 cancel()
188 if err != nil {
189 if given {
190 return c.fail(protocol.ExitUsage, "--api-base: %v", err)
191 }
192 return c.fail(protocol.ExitFailure, "%v", err)
193 }
194 site := siteFromAPI(apiBase)
195 host := strings.TrimPrefix(strings.TrimPrefix(site, "https://"), "http://")
196 // Accept a bare owner/repo or the repository's URL on that site.
197 from = strings.TrimPrefix(from, site+"/")
198 from = strings.TrimPrefix(from, host+"/")
199 from = strings.TrimSuffix(from, ".git")
200 if parts := strings.Split(from, "/"); len(parts) != 2 || parts[0] == "" || parts[1] == "" {
201 return c.fail(protocol.ExitUsage, "--from must be <owner>/<repo> (or the repository URL on %s)", site)
202 }
203 repo, code := resolveRepo(c, path, policy.CanWrite)
204 if code >= 0 {
205 return code
206 }
207 if code := refuseArchived(c, repo); code >= 0 {
208 return code
209 }
210 token := ""
211 if tokenStdin {
212 // Same discipline as repo import: token on stdin, never argv,
213 // never stored.
214 line, err := bufio.NewReader(c.Stdin).ReadString('\n')
215 if err != nil && line == "" {
216 return c.fail(protocol.ExitUsage, "--token-stdin: no token on stdin")
217 }
218 token = strings.TrimSpace(line)
219 }
220 g := &ghClient{base: apiBase, token: token, http: pinnedClient(api)}
221 g.detect()
222 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
223
224 // Pull heads first, so every merge request below has objects to point
225 // at. A mirror made with the default refspecs does not carry
226 // refs/pull/*, which is why imported pull requests used to have no
227 // head and `mr diff` could only fail on them (#128). Best-effort: an
228 // import of issues from a repository whose git data is not here yet
229 // is a legitimate thing to do, and the merge requests still arrive
230 // with their head SHA recorded.
231 fetchedPullHeads := fetchPullHeads(c, dir, site+"/"+from+".git", token)
232 src := host + "/" + from
233
234 var issues, mrs, comments, skipped, headed int
235 for page := 1; ; page++ {
236 var items []ghIssue
237 if err := g.get(g.issuesQuery(from, page), &items); err != nil {
238 return c.fail(protocol.ExitFailure, "%v", err)
239 }
240 if len(items) == 0 {
241 break
242 }
243 for _, it := range items {
244 key := fmt.Sprintf("gh:%d", it.Number)
245 val, seen, err := c.Store.ImportMarker(repo.ID, key)
246 if err != nil {
247 return c.fail(protocol.ExitFailure, "%v", err)
248 }
249 var localN int64
250 isPR := it.PullRequest != nil
251 if seen {
252 localN, _ = strconv.ParseInt(strings.TrimPrefix(strings.TrimPrefix(val, "issue:"), "mr:"), 10, 64)
253 skipped++
254 } else if isPR {
255 var pr ghPull
256 if err := g.get(fmt.Sprintf("/repos/%s/pulls/%d", from, it.Number), &pr); err != nil {
257 return c.fail(protocol.ExitFailure, "%v", err)
258 }
259 body := attribution(src, it.Number, "pull request", it.User.Login, it.CreatedAt) + it.Body
260 localN, err = c.Store.CreateMR(repo.ID, c.User.ID, repo.ID, pr.Head.Ref, pr.Base.Ref, it.Title, body, pr.Head.SHA, "md", false)
261 if err != nil {
262 return c.fail(protocol.ExitFailure, "%v", err)
263 }
264 mr, err := c.Store.MRByNumber(repo.ID, localN)
265 if err != nil {
266 return c.fail(protocol.ExitFailure, "%v", err)
267 }
268 if pr.MergedAt != "" {
269 c.Store.MarkMerged(mr.ID, pr.Base.SHA, 0, pr.MergedAt)
270 } else {
271 c.Store.MarkClosed(mr.ID, 0, it.ClosedAt)
272 }
273 // Point the MR head ref at the PR head, from the fetch
274 // above or from objects a mirror already had.
275 if pr.Head.SHA != "" && gitutil.HasCommit(dir, pr.Head.SHA) {
276 gitutil.UpdateRefCAS(dir, fmt.Sprintf("refs/merge-requests/%d/head", localN), pr.Head.SHA, "")
277 headed++
278 }
279 c.Store.SetImportMarker(repo.ID, key, fmt.Sprintf("mr:%d", localN))
280 mrs++
281 } else {
282 body := attribution(src, it.Number, "issue", it.User.Login, it.CreatedAt) + it.Body
283 localN, err = c.Store.CreateIssue(repo.ID, c.User.ID, it.Title, body, "md")
284 if err != nil {
285 return c.fail(protocol.ExitFailure, "%v", err)
286 }
287 iss, err := c.Store.IssueByNumber(repo.ID, localN)
288 if err != nil {
289 return c.fail(protocol.ExitFailure, "%v", err)
290 }
291 for _, l := range it.Labels {
292 c.Store.SetIssueLabel(repo, iss.ID, l.Name, true)
293 }
294 if it.State != "open" {
295 c.Store.SetIssueState(iss.ID, "closed")
296 }
297 c.Store.SetImportMarker(repo.ID, key, fmt.Sprintf("issue:%d", localN))
298 issues++
299 }
300 if it.Comments > 0 && localN > 0 {
301 n, err := importComments(c, g, repo, from, src, it.Number, localN, isPR)
302 if err != nil {
303 return c.fail(protocol.ExitFailure, "%v", err)
304 }
305 comments += n
306 }
307 fmt.Fprintf(c.Stderr, "%s#%d -> %s%d\n", src, it.Number, map[bool]string{true: "!", false: "#"}[isPR], localN)
308 }
309 }
310 d := map[string]any{"issues": issues, "mrs": mrs, "comments": comments,
311 "already_imported": skipped, "mrs_with_head": headed, "pull_heads_fetched": fetchedPullHeads}
312 return c.emit(d, func(w io.Writer) {
313 fmt.Fprintf(w, "imported %d issues, %d merge requests, %d comments (%d items already imported)\n",
314 issues, mrs, comments, skipped)
315 if mrs > headed {
316 fmt.Fprintf(w, "%d merge request(s) have no head objects; `mr diff` cannot render them.\n", mrs-headed)
317 if !fetchedPullHeads {
318 fmt.Fprintln(w, "refs/pull/* could not be fetched — re-run with --token-stdin if the repository is private.")
319 }
320 }
321 })
322}
323
324func importComments(c *Ctx, g *ghClient, repo store.Repo, from, src string, ghN, localN int64, isPR bool) (int, error) {
325 var localIssueID, localMRID int64
326 if isPR {
327 mr, err := c.Store.MRByNumber(repo.ID, localN)
328 if err != nil {
329 return 0, err
330 }
331 localMRID = mr.ID
332 } else {
333 iss, err := c.Store.IssueByNumber(repo.ID, localN)
334 if err != nil {
335 return 0, err
336 }
337 localIssueID = iss.ID
338 }
339 imported := 0
340 for page := 1; ; page++ {
341 // Forgejo returns every comment in one unpaged reply.
342 if g.forgejo && page > 1 {
343 return imported, nil
344 }
345 var cs []ghComment
346 q := fmt.Sprintf("/repos/%s/issues/%d/comments?per_page=100&page=%d", url.PathEscape(from), ghN, page)
347 q = strings.ReplaceAll(q, "%2F", "/")
348 if err := g.get(q, &cs); err != nil {
349 return imported, err
350 }
351 if len(cs) == 0 {
352 return imported, nil
353 }
354 for _, cm := range cs {
355 key := fmt.Sprintf("ghc:%d", cm.ID)
356 if _, seen, err := c.Store.ImportMarker(repo.ID, key); err != nil {
357 return imported, err
358 } else if seen {
359 continue
360 }
361 body := fmt.Sprintf("> @%s, %s\n\n%s", cm.User.Login, ghDate(cm.CreatedAt), cm.Body)
362 var err error
363 if isPR {
364 err = c.Store.AddMRComment(localMRID, c.User.ID, body, "md")
365 } else {
366 err = c.Store.AddIssueComment(localIssueID, c.User.ID, body, "md")
367 }
368 if err != nil {
369 return imported, err
370 }
371 c.Store.SetImportMarker(repo.ID, key, "")
372 imported++
373 }
374 }
375}
376
377// ghAskpass answers git's credential prompts from the environment, so a
378// token never appears in argv where /proc would expose it. Same shape the
379// mirror worker uses.
380const ghAskpass = `#!/bin/sh
381case "$1" in
382 Username*) echo "x-access-token" ;;
383 *) echo "${GITBAY_GH_TOKEN}" ;;
384esac
385`
386
387// fetchPullHeads brings refs/pull/*/head into refs/gh-pull/*; GitHub and
388// Forgejo both publish pull heads under that name. Reports whether it
389// worked; a failure is not fatal, since importing issues from a
390// repository whose git data is not here yet is a reasonable thing to do.
391// git connects only to the addresses the remote's host resolved to and
392// passed the check here, as repo import does (#298, #301).
393func fetchPullHeads(c *Ctx, dir, remote, token string) bool {
394 ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
395 defer cancel()
396 if err := gitpin.CheckGit(ctx); err != nil {
397 fmt.Fprintf(c.Stderr, "pull heads not fetched: %v\n", err)
398 return false
399 }
400 pinned, err := gitpin.Resolve(ctx, importLookup, remote, c.Cfg.Webhooks.AllowLocal)
401 if err != nil {
402 fmt.Fprintf(c.Stderr, "pull heads not fetched: %v\n", err)
403 return false
404 }
405 env := gitpin.Env(c.Cfg.Server.Root)
406 if token != "" {
407 askpass := filepath.Join(c.Cfg.Server.Root, "gh-import-askpass.sh")
408 if err := os.WriteFile(askpass, []byte(ghAskpass), 0o700); err != nil {
409 return false
410 }
411 env = append(env, "GIT_ASKPASS="+askpass, "GITBAY_GH_TOKEN="+token)
412 }
413 if err := gitutil.FetchPullHeads(ctx, dir, remote, io.Discard, pinned.Args(), env); err != nil {
414 return false
415 }
416 return true
417}