internal/control/ghimport.go
292 lines · 9202 bytes
1package control
2
3import (
4 "bufio"
5 "encoding/json"
6 "fmt"
7 "io"
8 "net/http"
9 "net/url"
10 "strconv"
11 "strings"
12 "time"
13
14 "gitbay.org/gitbay/internal/gitutil"
15 "gitbay.org/gitbay/internal/policy"
16 "gitbay.org/gitbay/internal/protocol"
17 "gitbay.org/gitbay/internal/store"
18 "gitbay.org/gitbay/internal/webhook"
19)
20
21func init() {
22 register(Command{Path: []string{"repo", "import-issues"},
23 Summary: "import GitHub issue and PR history",
24 Usage: "repo import-issues <owner/name> --from <ghowner/ghrepo> [--token-stdin] [--api-base <url>]",
25 ReadsStdin: true, Run: runImportIssues})
26}
27
28// GitHub API shapes, minimal.
29type ghUser struct {
30 Login string `json:"login"`
31}
32type ghIssue struct {
33 Number int64 `json:"number"`
34 Title string `json:"title"`
35 Body string `json:"body"`
36 State string `json:"state"`
37 CreatedAt string `json:"created_at"`
38 ClosedAt string `json:"closed_at"`
39 User ghUser `json:"user"`
40 Labels []struct{ Name string } `json:"labels"`
41 PullRequest *struct{} `json:"pull_request"`
42 Comments int `json:"comments"`
43}
44type ghPull struct {
45 MergedAt string `json:"merged_at"`
46 Head struct {
47 SHA string `json:"sha"`
48 Ref string `json:"ref"`
49 } `json:"head"`
50 Base struct {
51 SHA string `json:"sha"`
52 Ref string `json:"ref"`
53 } `json:"base"`
54}
55type ghComment struct {
56 ID int64 `json:"id"`
57 Body string `json:"body"`
58 CreatedAt string `json:"created_at"`
59 User ghUser `json:"user"`
60}
61
62type ghClient struct {
63 base string
64 token string
65 http *http.Client
66}
67
68func (g *ghClient) get(path string, out any) error {
69 req, err := http.NewRequest("GET", g.base+path, nil)
70 if err != nil {
71 return err
72 }
73 req.Header.Set("Accept", "application/vnd.github+json")
74 if g.token != "" {
75 req.Header.Set("Authorization", "Bearer "+g.token)
76 }
77 resp, err := g.http.Do(req)
78 if err != nil {
79 return err
80 }
81 defer resp.Body.Close()
82 if resp.StatusCode != 200 {
83 body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
84 return fmt.Errorf("GitHub API %s: %s: %.200s", path, resp.Status, body)
85 }
86 return json.NewDecoder(resp.Body).Decode(out)
87}
88
89func ghDate(iso string) string {
90 if t, err := time.Parse(time.RFC3339, iso); err == nil {
91 return t.UTC().Format("2006-01-02")
92 }
93 return iso
94}
95
96// attribution heads every imported body: foreign authors have no local
97// account, so the original author and date live in the text.
98func attribution(src string, n int64, kind, login, date string) string {
99 return fmt.Sprintf("> imported %s %s#%d — @%s, %s\n\n", kind, src, n, login, ghDate(date))
100}
101
102func runImportIssues(c *Ctx, args []string) int {
103 var path, from, apiBase string
104 tokenStdin := false
105 for i := 0; i < len(args); i++ {
106 switch args[i] {
107 case "--from", "--api-base":
108 if i+1 >= len(args) {
109 return c.fail(protocol.ExitUsage, "%s requires a value", args[i])
110 }
111 if args[i] == "--from" {
112 from = args[i+1]
113 } else {
114 apiBase = args[i+1]
115 }
116 i++
117 case "--token-stdin":
118 tokenStdin = true
119 default:
120 if path != "" {
121 return c.fail(protocol.ExitUsage, "unexpected argument %q", args[i])
122 }
123 path = args[i]
124 }
125 }
126 if path == "" || from == "" {
127 return c.fail(protocol.ExitUsage, "usage: repo import-issues <owner/name> --from <ghowner/ghrepo> [--token-stdin]")
128 }
129 // Accept a bare owner/repo or a full github.com URL.
130 from = strings.TrimSuffix(strings.TrimPrefix(strings.TrimPrefix(from, "https://"), "github.com/"), ".git")
131 if parts := strings.Split(from, "/"); len(parts) != 2 || parts[0] == "" || parts[1] == "" {
132 return c.fail(protocol.ExitUsage, "--from must be <ghowner>/<ghrepo> (or the github.com URL)")
133 }
134 if apiBase == "" {
135 apiBase = "https://api.github.com"
136 } else if err := webhook.ValidateURL(apiBase, c.Cfg.Webhooks.AllowLocal); err != nil {
137 // A writer-supplied API base is the same SSRF surface as a
138 // webhook target; same rules apply.
139 return c.fail(protocol.ExitUsage, "--api-base: %v", err)
140 }
141 repo, code := resolveRepo(c, path, policy.CanWrite)
142 if code >= 0 {
143 return code
144 }
145 if code := refuseArchived(c, repo); code >= 0 {
146 return code
147 }
148 token := ""
149 if tokenStdin {
150 // Same discipline as repo import: token on stdin, never argv,
151 // never stored.
152 line, err := bufio.NewReader(c.Stdin).ReadString('\n')
153 if err != nil && line == "" {
154 return c.fail(protocol.ExitUsage, "--token-stdin: no token on stdin")
155 }
156 token = strings.TrimSpace(line)
157 }
158 g := &ghClient{base: apiBase, token: token, http: &http.Client{Timeout: 30 * time.Second}}
159 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
160 src := "github.com/" + from
161
162 var issues, mrs, comments, skipped int
163 for page := 1; ; page++ {
164 var items []ghIssue
165 q := fmt.Sprintf("/repos/%s/issues?state=all&sort=created&direction=asc&per_page=100&page=%d", from, page)
166 if err := g.get(q, &items); err != nil {
167 return c.fail(protocol.ExitFailure, "%v", err)
168 }
169 if len(items) == 0 {
170 break
171 }
172 for _, it := range items {
173 key := fmt.Sprintf("gh:%d", it.Number)
174 val, seen, err := c.Store.ImportMarker(repo.ID, key)
175 if err != nil {
176 return c.fail(protocol.ExitFailure, "%v", err)
177 }
178 var localN int64
179 isPR := it.PullRequest != nil
180 if seen {
181 localN, _ = strconv.ParseInt(strings.TrimPrefix(strings.TrimPrefix(val, "issue:"), "mr:"), 10, 64)
182 skipped++
183 } else if isPR {
184 var pr ghPull
185 if err := g.get(fmt.Sprintf("/repos/%s/pulls/%d", from, it.Number), &pr); err != nil {
186 return c.fail(protocol.ExitFailure, "%v", err)
187 }
188 body := attribution(src, it.Number, "pull request", it.User.Login, it.CreatedAt) + it.Body
189 localN, err = c.Store.CreateMR(repo.ID, c.User.ID, repo.ID, pr.Head.Ref, pr.Base.Ref, it.Title, body, pr.Head.SHA, "md")
190 if err != nil {
191 return c.fail(protocol.ExitFailure, "%v", err)
192 }
193 mr, err := c.Store.MRByNumber(repo.ID, localN)
194 if err != nil {
195 return c.fail(protocol.ExitFailure, "%v", err)
196 }
197 if pr.MergedAt != "" {
198 c.Store.MarkMerged(mr.ID, pr.Base.SHA, 0, pr.MergedAt)
199 } else {
200 c.Store.MarkClosed(mr.ID, 0, it.ClosedAt)
201 }
202 // Point the MR head ref at the PR head when the mirror
203 // already holds the objects (refs/pull backups).
204 if pr.Head.SHA != "" && gitutil.HasCommit(dir, pr.Head.SHA) {
205 gitutil.UpdateRefCAS(dir, fmt.Sprintf("refs/merge-requests/%d/head", localN), pr.Head.SHA, "")
206 }
207 c.Store.SetImportMarker(repo.ID, key, fmt.Sprintf("mr:%d", localN))
208 mrs++
209 } else {
210 body := attribution(src, it.Number, "issue", it.User.Login, it.CreatedAt) + it.Body
211 localN, err = c.Store.CreateIssue(repo.ID, c.User.ID, it.Title, body, "md")
212 if err != nil {
213 return c.fail(protocol.ExitFailure, "%v", err)
214 }
215 iss, err := c.Store.IssueByNumber(repo.ID, localN)
216 if err != nil {
217 return c.fail(protocol.ExitFailure, "%v", err)
218 }
219 for _, l := range it.Labels {
220 c.Store.SetIssueLabel(repo.ID, iss.ID, l.Name, true)
221 }
222 if it.State != "open" {
223 c.Store.SetIssueState(iss.ID, "closed")
224 }
225 c.Store.SetImportMarker(repo.ID, key, fmt.Sprintf("issue:%d", localN))
226 issues++
227 }
228 if it.Comments > 0 && localN > 0 {
229 n, err := importComments(c, g, repo, from, src, it.Number, localN, isPR)
230 if err != nil {
231 return c.fail(protocol.ExitFailure, "%v", err)
232 }
233 comments += n
234 }
235 fmt.Fprintf(c.Stderr, "%s#%d -> %s%d\n", src, it.Number, map[bool]string{true: "!", false: "#"}[isPR], localN)
236 }
237 }
238 d := map[string]any{"issues": issues, "mrs": mrs, "comments": comments, "already_imported": skipped}
239 return c.emit(d, func(w io.Writer) {
240 fmt.Fprintf(w, "imported %d issues, %d merge requests, %d comments (%d items already imported)\n",
241 issues, mrs, comments, skipped)
242 })
243}
244
245func importComments(c *Ctx, g *ghClient, repo store.Repo, from, src string, ghN, localN int64, isPR bool) (int, error) {
246 var localIssueID, localMRID int64
247 if isPR {
248 mr, err := c.Store.MRByNumber(repo.ID, localN)
249 if err != nil {
250 return 0, err
251 }
252 localMRID = mr.ID
253 } else {
254 iss, err := c.Store.IssueByNumber(repo.ID, localN)
255 if err != nil {
256 return 0, err
257 }
258 localIssueID = iss.ID
259 }
260 imported := 0
261 for page := 1; ; page++ {
262 var cs []ghComment
263 q := fmt.Sprintf("/repos/%s/issues/%d/comments?per_page=100&page=%d", url.PathEscape(from), ghN, page)
264 q = strings.ReplaceAll(q, "%2F", "/")
265 if err := g.get(q, &cs); err != nil {
266 return imported, err
267 }
268 if len(cs) == 0 {
269 return imported, nil
270 }
271 for _, cm := range cs {
272 key := fmt.Sprintf("ghc:%d", cm.ID)
273 if _, seen, err := c.Store.ImportMarker(repo.ID, key); err != nil {
274 return imported, err
275 } else if seen {
276 continue
277 }
278 body := fmt.Sprintf("> @%s, %s\n\n%s", cm.User.Login, ghDate(cm.CreatedAt), cm.Body)
279 var err error
280 if isPR {
281 err = c.Store.AddMRComment(localMRID, c.User.ID, body, "md")
282 } else {
283 err = c.Store.AddIssueComment(localIssueID, c.User.ID, body, "md")
284 }
285 if err != nil {
286 return imported, err
287 }
288 c.Store.SetImportMarker(repo.ID, key, "")
289 imported++
290 }
291 }
292}