internal/control/ghimport.go
327 lines · 11164 bytes
1package control
2
3import (
4 "bufio"
5 "context"
6 "encoding/json"
7 "fmt"
8 "io"
9 "net/http"
10 "net/url"
11 "os"
12 "path/filepath"
13 "strconv"
14 "strings"
15 "time"
16
17 "gitbay.org/gitbay/internal/gitutil"
18 "gitbay.org/gitbay/internal/policy"
19 "gitbay.org/gitbay/internal/protocol"
20 "gitbay.org/gitbay/internal/store"
21 "gitbay.org/gitbay/internal/webhook"
22)
23
24func init() {
25 register(Command{Path: []string{"repo", "import-issues"},
26 Summary: "import GitHub issue and PR history",
27 Usage: "repo import-issues <owner/name> --from <ghowner/ghrepo> [--token-stdin] [--api-base <url>]",
28 ReadsStdin: true, Run: runImportIssues})
29}
30
31// GitHub API shapes, minimal.
32type ghUser struct {
33 Login string `json:"login"`
34}
35type ghIssue struct {
36 Number int64 `json:"number"`
37 Title string `json:"title"`
38 Body string `json:"body"`
39 State string `json:"state"`
40 CreatedAt string `json:"created_at"`
41 ClosedAt string `json:"closed_at"`
42 User ghUser `json:"user"`
43 Labels []struct{ Name string } `json:"labels"`
44 PullRequest *struct{} `json:"pull_request"`
45 Comments int `json:"comments"`
46}
47type ghPull struct {
48 MergedAt string `json:"merged_at"`
49 Head struct {
50 SHA string `json:"sha"`
51 Ref string `json:"ref"`
52 } `json:"head"`
53 Base struct {
54 SHA string `json:"sha"`
55 Ref string `json:"ref"`
56 } `json:"base"`
57}
58type ghComment struct {
59 ID int64 `json:"id"`
60 Body string `json:"body"`
61 CreatedAt string `json:"created_at"`
62 User ghUser `json:"user"`
63}
64
65type ghClient struct {
66 base string
67 token string
68 http *http.Client
69}
70
71func (g *ghClient) get(path string, out any) error {
72 req, err := http.NewRequest("GET", g.base+path, nil)
73 if err != nil {
74 return err
75 }
76 req.Header.Set("Accept", "application/vnd.github+json")
77 if g.token != "" {
78 req.Header.Set("Authorization", "Bearer "+g.token)
79 }
80 resp, err := g.http.Do(req)
81 if err != nil {
82 return err
83 }
84 defer resp.Body.Close()
85 if resp.StatusCode != 200 {
86 body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
87 return fmt.Errorf("GitHub API %s: %s: %.200s", path, resp.Status, body)
88 }
89 return json.NewDecoder(resp.Body).Decode(out)
90}
91
92func ghDate(iso string) string {
93 if t, err := time.Parse(time.RFC3339, iso); err == nil {
94 return t.UTC().Format("2006-01-02")
95 }
96 return iso
97}
98
99// attribution heads every imported body: foreign authors have no local
100// account, so the original author and date live in the text.
101func attribution(src string, n int64, kind, login, date string) string {
102 return fmt.Sprintf("> imported %s %s#%d — @%s, %s\n\n", kind, src, n, login, ghDate(date))
103}
104
105func runImportIssues(c *Ctx, args []string) int {
106 f, err := parseFlags(args, flagSpec{Values: []string{"--from", "--api-base"}, Bools: []string{"--token-stdin"}, MaxPos: 1,
107 Usage: "repo import-issues <owner/name> --from <owner/repo> [--api-base <url>] [--token-stdin]"})
108 if err != nil {
109 return c.fail(protocol.ExitUsage, "%v", err)
110 }
111 path, from, apiBase, tokenStdin := f.pos(0), f.Value("--from"), f.Value("--api-base"), f.Has("--token-stdin")
112 if path == "" || from == "" {
113 return c.fail(protocol.ExitUsage, "usage: repo import-issues <owner/name> --from <ghowner/ghrepo> [--token-stdin]")
114 }
115 // Accept a bare owner/repo or a full github.com URL.
116 from = strings.TrimSuffix(strings.TrimPrefix(strings.TrimPrefix(from, "https://"), "github.com/"), ".git")
117 if parts := strings.Split(from, "/"); len(parts) != 2 || parts[0] == "" || parts[1] == "" {
118 return c.fail(protocol.ExitUsage, "--from must be <ghowner>/<ghrepo> (or the github.com URL)")
119 }
120 if apiBase == "" {
121 apiBase = "https://api.github.com"
122 } else if err := webhook.ValidateURL(apiBase, c.Cfg.Webhooks.AllowLocal); err != nil {
123 // A writer-supplied API base is the same SSRF surface as a
124 // webhook target; same rules apply.
125 return c.fail(protocol.ExitUsage, "--api-base: %v", err)
126 }
127 repo, code := resolveRepo(c, path, policy.CanWrite)
128 if code >= 0 {
129 return code
130 }
131 if code := refuseArchived(c, repo); code >= 0 {
132 return code
133 }
134 token := ""
135 if tokenStdin {
136 // Same discipline as repo import: token on stdin, never argv,
137 // never stored.
138 line, err := bufio.NewReader(c.Stdin).ReadString('\n')
139 if err != nil && line == "" {
140 return c.fail(protocol.ExitUsage, "--token-stdin: no token on stdin")
141 }
142 token = strings.TrimSpace(line)
143 }
144 g := &ghClient{base: apiBase, token: token, http: &http.Client{Timeout: 30 * time.Second}}
145 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
146
147 // Pull heads first, so every merge request below has objects to point
148 // at. A mirror made with the default refspecs does not carry
149 // refs/pull/*, which is why imported pull requests used to have no
150 // head and `mr diff` could only fail on them (#128). Best-effort: an
151 // import of issues from a repository whose git data is not here yet
152 // is a legitimate thing to do, and the merge requests still arrive
153 // with their head SHA recorded.
154 fetchedPullHeads := fetchPullHeads(c, dir, from, token)
155 src := "github.com/" + from
156
157 var issues, mrs, comments, skipped, headed int
158 for page := 1; ; page++ {
159 var items []ghIssue
160 q := fmt.Sprintf("/repos/%s/issues?state=all&sort=created&direction=asc&per_page=100&page=%d", from, page)
161 if err := g.get(q, &items); err != nil {
162 return c.fail(protocol.ExitFailure, "%v", err)
163 }
164 if len(items) == 0 {
165 break
166 }
167 for _, it := range items {
168 key := fmt.Sprintf("gh:%d", it.Number)
169 val, seen, err := c.Store.ImportMarker(repo.ID, key)
170 if err != nil {
171 return c.fail(protocol.ExitFailure, "%v", err)
172 }
173 var localN int64
174 isPR := it.PullRequest != nil
175 if seen {
176 localN, _ = strconv.ParseInt(strings.TrimPrefix(strings.TrimPrefix(val, "issue:"), "mr:"), 10, 64)
177 skipped++
178 } else if isPR {
179 var pr ghPull
180 if err := g.get(fmt.Sprintf("/repos/%s/pulls/%d", from, it.Number), &pr); err != nil {
181 return c.fail(protocol.ExitFailure, "%v", err)
182 }
183 body := attribution(src, it.Number, "pull request", it.User.Login, it.CreatedAt) + it.Body
184 localN, err = c.Store.CreateMR(repo.ID, c.User.ID, repo.ID, pr.Head.Ref, pr.Base.Ref, it.Title, body, pr.Head.SHA, "md", false)
185 if err != nil {
186 return c.fail(protocol.ExitFailure, "%v", err)
187 }
188 mr, err := c.Store.MRByNumber(repo.ID, localN)
189 if err != nil {
190 return c.fail(protocol.ExitFailure, "%v", err)
191 }
192 if pr.MergedAt != "" {
193 c.Store.MarkMerged(mr.ID, pr.Base.SHA, 0, pr.MergedAt)
194 } else {
195 c.Store.MarkClosed(mr.ID, 0, it.ClosedAt)
196 }
197 // Point the MR head ref at the PR head, from the fetch
198 // above or from objects a mirror already had.
199 if pr.Head.SHA != "" && gitutil.HasCommit(dir, pr.Head.SHA) {
200 gitutil.UpdateRefCAS(dir, fmt.Sprintf("refs/merge-requests/%d/head", localN), pr.Head.SHA, "")
201 headed++
202 }
203 c.Store.SetImportMarker(repo.ID, key, fmt.Sprintf("mr:%d", localN))
204 mrs++
205 } else {
206 body := attribution(src, it.Number, "issue", it.User.Login, it.CreatedAt) + it.Body
207 localN, err = c.Store.CreateIssue(repo.ID, c.User.ID, it.Title, body, "md")
208 if err != nil {
209 return c.fail(protocol.ExitFailure, "%v", err)
210 }
211 iss, err := c.Store.IssueByNumber(repo.ID, localN)
212 if err != nil {
213 return c.fail(protocol.ExitFailure, "%v", err)
214 }
215 for _, l := range it.Labels {
216 c.Store.SetIssueLabel(repo.ID, iss.ID, l.Name, true)
217 }
218 if it.State != "open" {
219 c.Store.SetIssueState(iss.ID, "closed")
220 }
221 c.Store.SetImportMarker(repo.ID, key, fmt.Sprintf("issue:%d", localN))
222 issues++
223 }
224 if it.Comments > 0 && localN > 0 {
225 n, err := importComments(c, g, repo, from, src, it.Number, localN, isPR)
226 if err != nil {
227 return c.fail(protocol.ExitFailure, "%v", err)
228 }
229 comments += n
230 }
231 fmt.Fprintf(c.Stderr, "%s#%d -> %s%d\n", src, it.Number, map[bool]string{true: "!", false: "#"}[isPR], localN)
232 }
233 }
234 d := map[string]any{"issues": issues, "mrs": mrs, "comments": comments,
235 "already_imported": skipped, "mrs_with_head": headed, "pull_heads_fetched": fetchedPullHeads}
236 return c.emit(d, func(w io.Writer) {
237 fmt.Fprintf(w, "imported %d issues, %d merge requests, %d comments (%d items already imported)\n",
238 issues, mrs, comments, skipped)
239 if mrs > headed {
240 fmt.Fprintf(w, "%d merge request(s) have no head objects; `mr diff` cannot render them.\n", mrs-headed)
241 if !fetchedPullHeads {
242 fmt.Fprintln(w, "refs/pull/* could not be fetched — re-run with --token-stdin if the repository is private.")
243 }
244 }
245 })
246}
247
248func importComments(c *Ctx, g *ghClient, repo store.Repo, from, src string, ghN, localN int64, isPR bool) (int, error) {
249 var localIssueID, localMRID int64
250 if isPR {
251 mr, err := c.Store.MRByNumber(repo.ID, localN)
252 if err != nil {
253 return 0, err
254 }
255 localMRID = mr.ID
256 } else {
257 iss, err := c.Store.IssueByNumber(repo.ID, localN)
258 if err != nil {
259 return 0, err
260 }
261 localIssueID = iss.ID
262 }
263 imported := 0
264 for page := 1; ; page++ {
265 var cs []ghComment
266 q := fmt.Sprintf("/repos/%s/issues/%d/comments?per_page=100&page=%d", url.PathEscape(from), ghN, page)
267 q = strings.ReplaceAll(q, "%2F", "/")
268 if err := g.get(q, &cs); err != nil {
269 return imported, err
270 }
271 if len(cs) == 0 {
272 return imported, nil
273 }
274 for _, cm := range cs {
275 key := fmt.Sprintf("ghc:%d", cm.ID)
276 if _, seen, err := c.Store.ImportMarker(repo.ID, key); err != nil {
277 return imported, err
278 } else if seen {
279 continue
280 }
281 body := fmt.Sprintf("> @%s, %s\n\n%s", cm.User.Login, ghDate(cm.CreatedAt), cm.Body)
282 var err error
283 if isPR {
284 err = c.Store.AddMRComment(localMRID, c.User.ID, body, "md")
285 } else {
286 err = c.Store.AddIssueComment(localIssueID, c.User.ID, body, "md")
287 }
288 if err != nil {
289 return imported, err
290 }
291 c.Store.SetImportMarker(repo.ID, key, "")
292 imported++
293 }
294 }
295}
296
297// ghAskpass answers git's credential prompts from the environment, so a
298// token never appears in argv where /proc would expose it. Same shape the
299// mirror worker uses.
300const ghAskpass = `#!/bin/sh
301case "$1" in
302 Username*) echo "x-access-token" ;;
303 *) echo "${GITBAY_GH_TOKEN}" ;;
304esac
305`
306
307// fetchPullHeads brings refs/pull/*/head into refs/gh-pull/*. Reports
308// whether it worked; a failure is not fatal, since importing issues from
309// a repository whose git data is not here yet is a reasonable thing to
310// do.
311func fetchPullHeads(c *Ctx, dir, from, token string) bool {
312 env := []string{"GIT_TERMINAL_PROMPT=0", "HOME=" + c.Cfg.Server.Root}
313 if token != "" {
314 askpass := filepath.Join(c.Cfg.Server.Root, "gh-import-askpass.sh")
315 if err := os.WriteFile(askpass, []byte(ghAskpass), 0o700); err != nil {
316 return false
317 }
318 env = append(env, "GIT_ASKPASS="+askpass, "GITBAY_GH_TOKEN="+token)
319 }
320 ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
321 defer cancel()
322 url := "https://github.com/" + from + ".git"
323 if err := gitutil.FetchPullHeads(ctx, dir, url, io.Discard, env); err != nil {
324 return false
325 }
326 return true
327}