internal/control/ghimport.go
275 lines · 9092 bytes
1package control
2
3import (
4 "bufio"
5 "encoding/json"
6 "fmt"
7 "io"
8 "net/http"
9 "net/url"
10 "strconv"
11 "strings"
12 "time"
13
14 "gitbay.org/gitbay/internal/gitutil"
15 "gitbay.org/gitbay/internal/policy"
16 "gitbay.org/gitbay/internal/protocol"
17 "gitbay.org/gitbay/internal/store"
18 "gitbay.org/gitbay/internal/webhook"
19)
20
21func init() {
22 register(Command{Path: []string{"repo", "import-issues"},
23 Summary: "import GitHub issue and PR history",
24 Usage: "repo import-issues <owner/name> --from <ghowner/ghrepo> [--token-stdin] [--api-base <url>]",
25 ReadsStdin: true, Run: runImportIssues})
26}
27
28// GitHub API shapes, minimal.
29type ghUser struct {
30 Login string `json:"login"`
31}
32type ghIssue struct {
33 Number int64 `json:"number"`
34 Title string `json:"title"`
35 Body string `json:"body"`
36 State string `json:"state"`
37 CreatedAt string `json:"created_at"`
38 ClosedAt string `json:"closed_at"`
39 User ghUser `json:"user"`
40 Labels []struct{ Name string } `json:"labels"`
41 PullRequest *struct{} `json:"pull_request"`
42 Comments int `json:"comments"`
43}
44type ghPull struct {
45 MergedAt string `json:"merged_at"`
46 Head struct {
47 SHA string `json:"sha"`
48 Ref string `json:"ref"`
49 } `json:"head"`
50 Base struct {
51 SHA string `json:"sha"`
52 Ref string `json:"ref"`
53 } `json:"base"`
54}
55type ghComment struct {
56 ID int64 `json:"id"`
57 Body string `json:"body"`
58 CreatedAt string `json:"created_at"`
59 User ghUser `json:"user"`
60}
61
62type ghClient struct {
63 base string
64 token string
65 http *http.Client
66}
67
68func (g *ghClient) get(path string, out any) error {
69 req, err := http.NewRequest("GET", g.base+path, nil)
70 if err != nil {
71 return err
72 }
73 req.Header.Set("Accept", "application/vnd.github+json")
74 if g.token != "" {
75 req.Header.Set("Authorization", "Bearer "+g.token)
76 }
77 resp, err := g.http.Do(req)
78 if err != nil {
79 return err
80 }
81 defer resp.Body.Close()
82 if resp.StatusCode != 200 {
83 body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
84 return fmt.Errorf("GitHub API %s: %s: %.200s", path, resp.Status, body)
85 }
86 return json.NewDecoder(resp.Body).Decode(out)
87}
88
89func ghDate(iso string) string {
90 if t, err := time.Parse(time.RFC3339, iso); err == nil {
91 return t.UTC().Format("2006-01-02")
92 }
93 return iso
94}
95
96// attribution heads every imported body: foreign authors have no local
97// account, so the original author and date live in the text.
98func attribution(src string, n int64, kind, login, date string) string {
99 return fmt.Sprintf("> imported %s %s#%d — @%s, %s\n\n", kind, src, n, login, ghDate(date))
100}
101
102func runImportIssues(c *Ctx, args []string) int {
103 f, err := parseFlags(args, flagSpec{Values: []string{"--from", "--api-base"}, Bools: []string{"--token-stdin"}, MaxPos: 1,
104 Usage: "repo import-issues <owner/name> --from <owner/repo> [--api-base <url>] [--token-stdin]"})
105 if err != nil {
106 return c.fail(protocol.ExitUsage, "%v", err)
107 }
108 path, from, apiBase, tokenStdin := f.pos(0), f.Value("--from"), f.Value("--api-base"), f.Has("--token-stdin")
109 if path == "" || from == "" {
110 return c.fail(protocol.ExitUsage, "usage: repo import-issues <owner/name> --from <ghowner/ghrepo> [--token-stdin]")
111 }
112 // Accept a bare owner/repo or a full github.com URL.
113 from = strings.TrimSuffix(strings.TrimPrefix(strings.TrimPrefix(from, "https://"), "github.com/"), ".git")
114 if parts := strings.Split(from, "/"); len(parts) != 2 || parts[0] == "" || parts[1] == "" {
115 return c.fail(protocol.ExitUsage, "--from must be <ghowner>/<ghrepo> (or the github.com URL)")
116 }
117 if apiBase == "" {
118 apiBase = "https://api.github.com"
119 } else if err := webhook.ValidateURL(apiBase, c.Cfg.Webhooks.AllowLocal); err != nil {
120 // A writer-supplied API base is the same SSRF surface as a
121 // webhook target; same rules apply.
122 return c.fail(protocol.ExitUsage, "--api-base: %v", err)
123 }
124 repo, code := resolveRepo(c, path, policy.CanWrite)
125 if code >= 0 {
126 return code
127 }
128 if code := refuseArchived(c, repo); code >= 0 {
129 return code
130 }
131 token := ""
132 if tokenStdin {
133 // Same discipline as repo import: token on stdin, never argv,
134 // never stored.
135 line, err := bufio.NewReader(c.Stdin).ReadString('\n')
136 if err != nil && line == "" {
137 return c.fail(protocol.ExitUsage, "--token-stdin: no token on stdin")
138 }
139 token = strings.TrimSpace(line)
140 }
141 g := &ghClient{base: apiBase, token: token, http: &http.Client{Timeout: 30 * time.Second}}
142 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
143 src := "github.com/" + from
144
145 var issues, mrs, comments, skipped int
146 for page := 1; ; page++ {
147 var items []ghIssue
148 q := fmt.Sprintf("/repos/%s/issues?state=all&sort=created&direction=asc&per_page=100&page=%d", from, page)
149 if err := g.get(q, &items); err != nil {
150 return c.fail(protocol.ExitFailure, "%v", err)
151 }
152 if len(items) == 0 {
153 break
154 }
155 for _, it := range items {
156 key := fmt.Sprintf("gh:%d", it.Number)
157 val, seen, err := c.Store.ImportMarker(repo.ID, key)
158 if err != nil {
159 return c.fail(protocol.ExitFailure, "%v", err)
160 }
161 var localN int64
162 isPR := it.PullRequest != nil
163 if seen {
164 localN, _ = strconv.ParseInt(strings.TrimPrefix(strings.TrimPrefix(val, "issue:"), "mr:"), 10, 64)
165 skipped++
166 } else if isPR {
167 var pr ghPull
168 if err := g.get(fmt.Sprintf("/repos/%s/pulls/%d", from, it.Number), &pr); err != nil {
169 return c.fail(protocol.ExitFailure, "%v", err)
170 }
171 body := attribution(src, it.Number, "pull request", it.User.Login, it.CreatedAt) + it.Body
172 localN, err = c.Store.CreateMR(repo.ID, c.User.ID, repo.ID, pr.Head.Ref, pr.Base.Ref, it.Title, body, pr.Head.SHA, "md")
173 if err != nil {
174 return c.fail(protocol.ExitFailure, "%v", err)
175 }
176 mr, err := c.Store.MRByNumber(repo.ID, localN)
177 if err != nil {
178 return c.fail(protocol.ExitFailure, "%v", err)
179 }
180 if pr.MergedAt != "" {
181 c.Store.MarkMerged(mr.ID, pr.Base.SHA, 0, pr.MergedAt)
182 } else {
183 c.Store.MarkClosed(mr.ID, 0, it.ClosedAt)
184 }
185 // Point the MR head ref at the PR head when the mirror
186 // already holds the objects (refs/pull backups).
187 if pr.Head.SHA != "" && gitutil.HasCommit(dir, pr.Head.SHA) {
188 gitutil.UpdateRefCAS(dir, fmt.Sprintf("refs/merge-requests/%d/head", localN), pr.Head.SHA, "")
189 }
190 c.Store.SetImportMarker(repo.ID, key, fmt.Sprintf("mr:%d", localN))
191 mrs++
192 } else {
193 body := attribution(src, it.Number, "issue", it.User.Login, it.CreatedAt) + it.Body
194 localN, err = c.Store.CreateIssue(repo.ID, c.User.ID, it.Title, body, "md")
195 if err != nil {
196 return c.fail(protocol.ExitFailure, "%v", err)
197 }
198 iss, err := c.Store.IssueByNumber(repo.ID, localN)
199 if err != nil {
200 return c.fail(protocol.ExitFailure, "%v", err)
201 }
202 for _, l := range it.Labels {
203 c.Store.SetIssueLabel(repo.ID, iss.ID, l.Name, true)
204 }
205 if it.State != "open" {
206 c.Store.SetIssueState(iss.ID, "closed")
207 }
208 c.Store.SetImportMarker(repo.ID, key, fmt.Sprintf("issue:%d", localN))
209 issues++
210 }
211 if it.Comments > 0 && localN > 0 {
212 n, err := importComments(c, g, repo, from, src, it.Number, localN, isPR)
213 if err != nil {
214 return c.fail(protocol.ExitFailure, "%v", err)
215 }
216 comments += n
217 }
218 fmt.Fprintf(c.Stderr, "%s#%d -> %s%d\n", src, it.Number, map[bool]string{true: "!", false: "#"}[isPR], localN)
219 }
220 }
221 d := map[string]any{"issues": issues, "mrs": mrs, "comments": comments, "already_imported": skipped}
222 return c.emit(d, func(w io.Writer) {
223 fmt.Fprintf(w, "imported %d issues, %d merge requests, %d comments (%d items already imported)\n",
224 issues, mrs, comments, skipped)
225 })
226}
227
228func importComments(c *Ctx, g *ghClient, repo store.Repo, from, src string, ghN, localN int64, isPR bool) (int, error) {
229 var localIssueID, localMRID int64
230 if isPR {
231 mr, err := c.Store.MRByNumber(repo.ID, localN)
232 if err != nil {
233 return 0, err
234 }
235 localMRID = mr.ID
236 } else {
237 iss, err := c.Store.IssueByNumber(repo.ID, localN)
238 if err != nil {
239 return 0, err
240 }
241 localIssueID = iss.ID
242 }
243 imported := 0
244 for page := 1; ; page++ {
245 var cs []ghComment
246 q := fmt.Sprintf("/repos/%s/issues/%d/comments?per_page=100&page=%d", url.PathEscape(from), ghN, page)
247 q = strings.ReplaceAll(q, "%2F", "/")
248 if err := g.get(q, &cs); err != nil {
249 return imported, err
250 }
251 if len(cs) == 0 {
252 return imported, nil
253 }
254 for _, cm := range cs {
255 key := fmt.Sprintf("ghc:%d", cm.ID)
256 if _, seen, err := c.Store.ImportMarker(repo.ID, key); err != nil {
257 return imported, err
258 } else if seen {
259 continue
260 }
261 body := fmt.Sprintf("> @%s, %s\n\n%s", cm.User.Login, ghDate(cm.CreatedAt), cm.Body)
262 var err error
263 if isPR {
264 err = c.Store.AddMRComment(localMRID, c.User.ID, body, "md")
265 } else {
266 err = c.Store.AddIssueComment(localIssueID, c.User.ID, body, "md")
267 }
268 if err != nil {
269 return imported, err
270 }
271 c.Store.SetImportMarker(repo.ID, key, "")
272 imported++
273 }
274 }
275}