internal/control/ghimport.go
291 lines · 9124 bytes
1package control
2
3import (
4 "bufio"
5 "encoding/json"
6 "fmt"
7 "io"
8 "net/http"
9 "net/url"
10 "strconv"
11 "strings"
12 "time"
13
14 "gitbay.org/gitbay/internal/gitutil"
15 "gitbay.org/gitbay/internal/policy"
16 "gitbay.org/gitbay/internal/protocol"
17 "gitbay.org/gitbay/internal/store"
18 "gitbay.org/gitbay/internal/webhook"
19)
20
21func init() {
22 register(Command{Path: []string{"repo", "import-issues"},
23 Summary: "import GitHub issue and PR history",
24 Usage: "repo import-issues <owner/name> --from <ghowner/ghrepo> [--token-stdin] [--api-base <url>]",
25 ReadsStdin: true, Run: runImportIssues})
26}
27
28// GitHub API shapes, minimal.
29type ghUser struct {
30 Login string `json:"login"`
31}
32type ghIssue struct {
33 Number int64 `json:"number"`
34 Title string `json:"title"`
35 Body string `json:"body"`
36 State string `json:"state"`
37 CreatedAt string `json:"created_at"`
38 User ghUser `json:"user"`
39 Labels []struct{ Name string } `json:"labels"`
40 PullRequest *struct{} `json:"pull_request"`
41 Comments int `json:"comments"`
42}
43type ghPull struct {
44 MergedAt string `json:"merged_at"`
45 Head struct {
46 SHA string `json:"sha"`
47 Ref string `json:"ref"`
48 } `json:"head"`
49 Base struct {
50 SHA string `json:"sha"`
51 Ref string `json:"ref"`
52 } `json:"base"`
53}
54type ghComment struct {
55 ID int64 `json:"id"`
56 Body string `json:"body"`
57 CreatedAt string `json:"created_at"`
58 User ghUser `json:"user"`
59}
60
61type ghClient struct {
62 base string
63 token string
64 http *http.Client
65}
66
67func (g *ghClient) get(path string, out any) error {
68 req, err := http.NewRequest("GET", g.base+path, nil)
69 if err != nil {
70 return err
71 }
72 req.Header.Set("Accept", "application/vnd.github+json")
73 if g.token != "" {
74 req.Header.Set("Authorization", "Bearer "+g.token)
75 }
76 resp, err := g.http.Do(req)
77 if err != nil {
78 return err
79 }
80 defer resp.Body.Close()
81 if resp.StatusCode != 200 {
82 body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
83 return fmt.Errorf("GitHub API %s: %s: %.200s", path, resp.Status, body)
84 }
85 return json.NewDecoder(resp.Body).Decode(out)
86}
87
88func ghDate(iso string) string {
89 if t, err := time.Parse(time.RFC3339, iso); err == nil {
90 return t.UTC().Format("2006-01-02")
91 }
92 return iso
93}
94
95// attribution heads every imported body: foreign authors have no local
96// account, so the original author and date live in the text.
97func attribution(src string, n int64, kind, login, date string) string {
98 return fmt.Sprintf("> imported %s %s#%d — @%s, %s\n\n", kind, src, n, login, ghDate(date))
99}
100
101func runImportIssues(c *Ctx, args []string) int {
102 var path, from, apiBase string
103 tokenStdin := false
104 for i := 0; i < len(args); i++ {
105 switch args[i] {
106 case "--from", "--api-base":
107 if i+1 >= len(args) {
108 return c.fail(protocol.ExitUsage, "%s requires a value", args[i])
109 }
110 if args[i] == "--from" {
111 from = args[i+1]
112 } else {
113 apiBase = args[i+1]
114 }
115 i++
116 case "--token-stdin":
117 tokenStdin = true
118 default:
119 if path != "" {
120 return c.fail(protocol.ExitUsage, "unexpected argument %q", args[i])
121 }
122 path = args[i]
123 }
124 }
125 if path == "" || from == "" {
126 return c.fail(protocol.ExitUsage, "usage: repo import-issues <owner/name> --from <ghowner/ghrepo> [--token-stdin]")
127 }
128 // Accept a bare owner/repo or a full github.com URL.
129 from = strings.TrimSuffix(strings.TrimPrefix(strings.TrimPrefix(from, "https://"), "github.com/"), ".git")
130 if parts := strings.Split(from, "/"); len(parts) != 2 || parts[0] == "" || parts[1] == "" {
131 return c.fail(protocol.ExitUsage, "--from must be <ghowner>/<ghrepo> (or the github.com URL)")
132 }
133 if apiBase == "" {
134 apiBase = "https://api.github.com"
135 } else if err := webhook.ValidateURL(apiBase, c.Cfg.Webhooks.AllowLocal); err != nil {
136 // A writer-supplied API base is the same SSRF surface as a
137 // webhook target; same rules apply.
138 return c.fail(protocol.ExitUsage, "--api-base: %v", err)
139 }
140 repo, code := resolveRepo(c, path, policy.CanWrite)
141 if code >= 0 {
142 return code
143 }
144 if code := refuseArchived(c, repo); code >= 0 {
145 return code
146 }
147 token := ""
148 if tokenStdin {
149 // Same discipline as repo import: token on stdin, never argv,
150 // never stored.
151 line, err := bufio.NewReader(c.Stdin).ReadString('\n')
152 if err != nil && line == "" {
153 return c.fail(protocol.ExitUsage, "--token-stdin: no token on stdin")
154 }
155 token = strings.TrimSpace(line)
156 }
157 g := &ghClient{base: apiBase, token: token, http: &http.Client{Timeout: 30 * time.Second}}
158 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
159 src := "github.com/" + from
160
161 var issues, mrs, comments, skipped int
162 for page := 1; ; page++ {
163 var items []ghIssue
164 q := fmt.Sprintf("/repos/%s/issues?state=all&sort=created&direction=asc&per_page=100&page=%d", from, page)
165 if err := g.get(q, &items); err != nil {
166 return c.fail(protocol.ExitFailure, "%v", err)
167 }
168 if len(items) == 0 {
169 break
170 }
171 for _, it := range items {
172 key := fmt.Sprintf("gh:%d", it.Number)
173 val, seen, err := c.Store.ImportMarker(repo.ID, key)
174 if err != nil {
175 return c.fail(protocol.ExitFailure, "%v", err)
176 }
177 var localN int64
178 isPR := it.PullRequest != nil
179 if seen {
180 localN, _ = strconv.ParseInt(strings.TrimPrefix(strings.TrimPrefix(val, "issue:"), "mr:"), 10, 64)
181 skipped++
182 } else if isPR {
183 var pr ghPull
184 if err := g.get(fmt.Sprintf("/repos/%s/pulls/%d", from, it.Number), &pr); err != nil {
185 return c.fail(protocol.ExitFailure, "%v", err)
186 }
187 body := attribution(src, it.Number, "pull request", it.User.Login, it.CreatedAt) + it.Body
188 localN, err = c.Store.CreateMR(repo.ID, c.User.ID, repo.ID, pr.Head.Ref, pr.Base.Ref, it.Title, body, pr.Head.SHA, "md")
189 if err != nil {
190 return c.fail(protocol.ExitFailure, "%v", err)
191 }
192 mr, err := c.Store.MRByNumber(repo.ID, localN)
193 if err != nil {
194 return c.fail(protocol.ExitFailure, "%v", err)
195 }
196 if pr.MergedAt != "" {
197 c.Store.MarkMerged(mr.ID, pr.Base.SHA)
198 } else {
199 c.Store.SetMRState(mr.ID, "closed")
200 }
201 // Point the MR head ref at the PR head when the mirror
202 // already holds the objects (refs/pull backups).
203 if pr.Head.SHA != "" && gitutil.HasCommit(dir, pr.Head.SHA) {
204 gitutil.UpdateRefCAS(dir, fmt.Sprintf("refs/merge-requests/%d/head", localN), pr.Head.SHA, "")
205 }
206 c.Store.SetImportMarker(repo.ID, key, fmt.Sprintf("mr:%d", localN))
207 mrs++
208 } else {
209 body := attribution(src, it.Number, "issue", it.User.Login, it.CreatedAt) + it.Body
210 localN, err = c.Store.CreateIssue(repo.ID, c.User.ID, it.Title, body, "md")
211 if err != nil {
212 return c.fail(protocol.ExitFailure, "%v", err)
213 }
214 iss, err := c.Store.IssueByNumber(repo.ID, localN)
215 if err != nil {
216 return c.fail(protocol.ExitFailure, "%v", err)
217 }
218 for _, l := range it.Labels {
219 c.Store.SetIssueLabel(repo.ID, iss.ID, l.Name, true)
220 }
221 if it.State != "open" {
222 c.Store.SetIssueState(iss.ID, "closed")
223 }
224 c.Store.SetImportMarker(repo.ID, key, fmt.Sprintf("issue:%d", localN))
225 issues++
226 }
227 if it.Comments > 0 && localN > 0 {
228 n, err := importComments(c, g, repo, from, src, it.Number, localN, isPR)
229 if err != nil {
230 return c.fail(protocol.ExitFailure, "%v", err)
231 }
232 comments += n
233 }
234 fmt.Fprintf(c.Stderr, "%s#%d -> %s%d\n", src, it.Number, map[bool]string{true: "!", false: "#"}[isPR], localN)
235 }
236 }
237 d := map[string]any{"issues": issues, "mrs": mrs, "comments": comments, "already_imported": skipped}
238 return c.emit(d, func(w io.Writer) {
239 fmt.Fprintf(w, "imported %d issues, %d merge requests, %d comments (%d items already imported)\n",
240 issues, mrs, comments, skipped)
241 })
242}
243
244func importComments(c *Ctx, g *ghClient, repo store.Repo, from, src string, ghN, localN int64, isPR bool) (int, error) {
245 var localIssueID, localMRID int64
246 if isPR {
247 mr, err := c.Store.MRByNumber(repo.ID, localN)
248 if err != nil {
249 return 0, err
250 }
251 localMRID = mr.ID
252 } else {
253 iss, err := c.Store.IssueByNumber(repo.ID, localN)
254 if err != nil {
255 return 0, err
256 }
257 localIssueID = iss.ID
258 }
259 imported := 0
260 for page := 1; ; page++ {
261 var cs []ghComment
262 q := fmt.Sprintf("/repos/%s/issues/%d/comments?per_page=100&page=%d", url.PathEscape(from), ghN, page)
263 q = strings.ReplaceAll(q, "%2F", "/")
264 if err := g.get(q, &cs); err != nil {
265 return imported, err
266 }
267 if len(cs) == 0 {
268 return imported, nil
269 }
270 for _, cm := range cs {
271 key := fmt.Sprintf("ghc:%d", cm.ID)
272 if _, seen, err := c.Store.ImportMarker(repo.ID, key); err != nil {
273 return imported, err
274 } else if seen {
275 continue
276 }
277 body := fmt.Sprintf("> @%s, %s\n\n%s", cm.User.Login, ghDate(cm.CreatedAt), cm.Body)
278 var err error
279 if isPR {
280 err = c.Store.AddMRComment(localMRID, c.User.ID, body, "md")
281 } else {
282 err = c.Store.AddIssueComment(localIssueID, c.User.ID, body, "md")
283 }
284 if err != nil {
285 return imported, err
286 }
287 c.Store.SetImportMarker(repo.ID, key, "")
288 imported++
289 }
290 }
291}