krz/gitbay

import-issues reads Forgejo through --api-base !349

merged merged by cmc on 2026-09-08 05:55 UTC · krz/gitbay:codeberg-import into main

4 files changed, +189 −29

Layout: unified · split

.gitbay/wiki/FAQ.org +3 −3
@@ -9,9 +9,9 @@
9 when the one below merges, and refuses squash or rebase under a stack. 9 when the one below merges, and refuses squash or rebase under a stack.
10 No stack command; the workflow is branches. See 10 No stack command; the workflow is branches. See
11 [[Stacked-MRs][Stacked merge requests]]. 11 [[Stacked-MRs][Stacked merge requests]].
12- Moving from GitHub? :: =gitbay repo import= (git data) then 12- Moving from GitHub or Codeberg? :: =gitbay repo import= (git data)
13 =gitbay repo import-issues= (history); =repo mirror= covers the 13 then =gitbay repo import-issues= (history, from GitHub or any Forgejo
14 transition window. 14 =--api-base=); =repo mirror= covers the transition window.
15- What is not planned? :: A container or package registry (scope creep 15- What is not planned? :: A container or package registry (scope creep
16 away from "forge"; external registries integrate through CI), an 16 away from "forge"; external registries integrate through CI), an
17 email patch flow (revisit only if sourcehut-style demand appears), 17 email patch flow (revisit only if sourcehut-style demand appears),
.gitbay/wiki/Users.org +10 −5
@@ -207,17 +207,22 @@ gitbay repo archive you/project # read-only: pushes and issu
207gitbay repo unarchive you/project # writes refused, browsing intact 207gitbay repo unarchive you/project # writes refused, browsing intact
208#+end_src 208#+end_src
209 209
210Import from another forge — git data first, then optionally the GitHub 210Import from another forge — git data first, then optionally the issue
211issue and PR history (issues keep state/labels/comments; PRs land as 211and PR history from GitHub or any Forgejo instance such as Codeberg
212closed or merged MRs with their discussion; originals are attributed 212(issues keep state/labels/comments; PRs land as closed or merged MRs
213inline since foreign authors have no local account; re-running resumes 213with their discussion; originals are attributed inline since foreign
214where it stopped): 214authors have no local account; re-running resumes where it stopped):
215#+begin_src sh 215#+begin_src sh
216gitbay repo import you/mirror --from https://github.com/you/repo.git \ 216gitbay repo import you/mirror --from https://github.com/you/repo.git \
217 [--private] [--token-stdin] # token on stdin, never in the URL 217 [--private] [--token-stdin] # token on stdin, never in the URL
218gitbay repo import-issues you/mirror --from you/repo --token-stdin 218gitbay repo import-issues you/mirror --from you/repo --token-stdin
219gitbay repo import-issues you/mirror --from you/repo \
220 --api-base https://codeberg.org/api/v1 # Forgejo: the site's /api/v1
219#+end_src 221#+end_src
220 222
223Sourcehut has no API of that shape; =repo import= takes its git data
224and the todo.sr.ht tracker is not read.
225
221Moving between gitbay instances (no lock-in): run on the TARGET, with 226Moving between gitbay instances (no lock-in): run on the TARGET, with
222your key registered on both sides. Profile, repos with settings, 227your key registered on both sides. Profile, repos with settings,
223issues, MRs, and comments replay with attribution; git data mirrors 228issues, MRs, and comments replay with attribution; git data mirrors
e2e/ghimport_test.go +111 −2
@@ -92,6 +92,7 @@ func TestGitHubIssueImport(t *testing.T) {
92 mustGit(t, dir, env, "push", "-q", "origin", "main") 92 mustGit(t, dir, env, "push", "-q", "origin", "main")
93 93
94 gh := fakeGitHub(t) 94 gh := fakeGitHub(t)
95 host := strings.TrimPrefix(gh.URL, "http://")
95 out, errOut, code := inst.ssh(t, aliceKey, "sekrit\n", "repo", "import-issues", "alice/app", 96 out, errOut, code := inst.ssh(t, aliceKey, "sekrit\n", "repo", "import-issues", "alice/app",
96 "--from", "octo/legacy", "--token-stdin", "--api-base", gh.URL) 97 "--from", "octo/legacy", "--token-stdin", "--api-base", gh.URL)
97 if code != 0 { 98 if code != 0 {
@@ -105,7 +106,7 @@ func TestGitHubIssueImport(t *testing.T) {
105 out, _, _ = inst.ssh(t, aliceKey, "", "issue", "show", "alice/app", "1", "--json") 106 out, _, _ = inst.ssh(t, aliceKey, "", "issue", "show", "alice/app", "1", "--json")
106 if !strings.Contains(out, "old bug") || !strings.Contains(out, `"state":"closed"`) || 107 if !strings.Contains(out, "old bug") || !strings.Contains(out, `"state":"closed"`) ||
107 !strings.Contains(out, `"labels":["bug"]`) || 108 !strings.Contains(out, `"labels":["bug"]`) ||
108 !strings.Contains(out, "imported issue github.com/octo/legacy#1") || 109 !strings.Contains(out, "imported issue "+host+"/octo/legacy#1") ||
109 !strings.Contains(out, "@octofan, 2019-03-04") { 110 !strings.Contains(out, "@octofan, 2019-03-04") {
110 t.Fatalf("issue 1: %s", out) 111 t.Fatalf("issue 1: %s", out)
111 } 112 }
@@ -118,7 +119,7 @@ func TestGitHubIssueImport(t *testing.T) {
118 // MR !1 (GitHub PR #2): merged, discussion imported. 119 // MR !1 (GitHub PR #2): merged, discussion imported.
119 out, _, _ = inst.ssh(t, aliceKey, "", "mr", "show", "alice/app", "1", "--json") 120 out, _, _ = inst.ssh(t, aliceKey, "", "mr", "show", "alice/app", "1", "--json")
120 if !strings.Contains(out, "add feature") || !strings.Contains(out, `"state":"merged"`) || 121 if !strings.Contains(out, "add feature") || !strings.Contains(out, `"state":"merged"`) ||
121 !strings.Contains(out, "imported pull request github.com/octo/legacy#2") || 122 !strings.Contains(out, "imported pull request "+host+"/octo/legacy#2") ||
122 !strings.Contains(out, "nice patch") { 123 !strings.Contains(out, "nice patch") {
123 t.Fatalf("mr 1: %s", out) 124 t.Fatalf("mr 1: %s", out)
124 } 125 }
@@ -154,3 +155,111 @@ func TestGitHubImportSSRFGuard(t *testing.T) {
154 t.Fatalf("local api-base allowed: exit %d, %s", code, errOut) 155 t.Fatalf("local api-base allowed: exit %d, %s", code, errOut)
155 } 156 }
156} 157}
158
159// fakeForgejo serves the Forgejo shape of the same API under /api/v1:
160// GitHub's issue, pull and comment objects, but pages sized by `limit`,
161// order by `sort=oldest`, a /version endpoint, and a comments endpoint
162// that ignores `page` and returns everything every time.
163func fakeForgejo(t *testing.T) *httptest.Server {
164 t.Helper()
165 mux := http.NewServeMux()
166 mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
167 fmt.Fprint(w, `{"version":"9.0.0+gitea-1.22.0"}`)
168 })
169 mux.HandleFunc("/api/v1/repos/octo/legacy/issues", func(w http.ResponseWriter, r *http.Request) {
170 q := r.URL.Query()
171 if q.Get("page") != "1" {
172 fmt.Fprint(w, "[]")
173 return
174 }
175 items := []string{
176 `{"number":1,"title":"old bug","body":"it crashed","state":"closed",
177 "created_at":"2019-03-04T10:00:00+01:00","user":{"login":"octofan"},
178 "labels":[{"name":"bug"}],"comments":0}`,
179 `{"number":2,"title":"add feature","body":"the patch","state":"closed",
180 "created_at":"2020-06-01T10:00:00Z","user":{"login":"drive-by"},
181 "labels":[],"comments":1,"pull_request":{"merged":true}}`,
182 `{"number":3,"title":"still open","body":"discuss","state":"open",
183 "created_at":"2021-01-01T10:00:00Z","user":{"login":"octofan"},
184 "labels":[],"comments":2}`,
185 }
186 // Forgejo's default is newest first; only sort=oldest gives
187 // the order local numbering depends on.
188 if q.Get("sort") != "oldest" || q.Get("limit") == "" {
189 items[0], items[2] = items[2], items[0]
190 }
191 fmt.Fprint(w, "["+strings.Join(items, ",")+"]")
192 })
193 mux.HandleFunc("/api/v1/repos/octo/legacy/pulls/2", func(w http.ResponseWriter, r *http.Request) {
194 fmt.Fprint(w, `{"merged_at":"2020-06-02T10:00:00Z",
195 "head":{"sha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","ref":"feature"},
196 "base":{"sha":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","ref":"main"}}`)
197 })
198 comments := func(payload string) http.HandlerFunc {
199 return func(w http.ResponseWriter, r *http.Request) {
200 // No paging on this endpoint: the real one ignores `page`
201 // and returns everything, so a caller walking pages never
202 // stops. Answer a second page with an error so the test
203 // fails instead of hanging.
204 if p := r.URL.Query().Get("page"); p != "" && p != "1" {
205 http.Error(w, "unpaged endpoint asked for page "+p, 500)
206 return
207 }
208 fmt.Fprint(w, payload)
209 }
210 }
211 mux.HandleFunc("/api/v1/repos/octo/legacy/issues/2/comments", comments(
212 `[{"id":101,"body":"nice patch","created_at":"2020-06-01T11:00:00Z","user":{"login":"maintainer"}}]`))
213 mux.HandleFunc("/api/v1/repos/octo/legacy/issues/3/comments", comments(
214 `[{"id":102,"body":"me too","created_at":"2021-01-02T10:00:00Z","user":{"login":"other"}},
215 {"id":103,"body":"still happening","created_at":"2021-02-01T10:00:00Z","user":{"login":"octofan"}}]`))
216 srv := httptest.NewServer(mux)
217 t.Cleanup(srv.Close)
218 return srv
219}
220
221func TestForgejoIssueImport(t *testing.T) {
222 inst := startInstanceWith(t, "[webhooks]\nallow_local = true\n")
223 aliceKey := inst.newKey(t, "alice")
224 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
225 if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
226 t.Fatalf("repo create: %s", errOut)
227 }
228 fj := fakeForgejo(t)
229 host := strings.TrimPrefix(fj.URL, "http://")
230 // --from as the repository's URL on the site, the way a Codeberg
231 // user copies it from the address bar.
232 out, errOut, code := inst.ssh(t, aliceKey, "", "repo", "import-issues", "alice/app",
233 "--from", fj.URL+"/octo/legacy", "--api-base", fj.URL+"/api/v1")
234 if code != 0 {
235 t.Fatalf("import: %s", errOut)
236 }
237 if !strings.Contains(out, "imported 2 issues, 1 merge requests, 3 comments") {
238 t.Fatalf("summary: %s", out)
239 }
240 // Oldest first, attributed to the site the API base belongs to.
241 out, _, _ = inst.ssh(t, aliceKey, "", "issue", "show", "alice/app", "1", "--json")
242 if !strings.Contains(out, "old bug") || !strings.Contains(out, `"state":"closed"`) ||
243 !strings.Contains(out, `"labels":["bug"]`) ||
244 !strings.Contains(out, "imported issue "+host+"/octo/legacy#1") ||
245 !strings.Contains(out, "@octofan, 2019-03-04") {
246 t.Fatalf("issue 1: %s", out)
247 }
248 out, _, _ = inst.ssh(t, aliceKey, "", "issue", "show", "alice/app", "2", "--json")
249 if !strings.Contains(out, "still open") || !strings.Contains(out, "me too") ||
250 !strings.Contains(out, "still happening") {
251 t.Fatalf("issue 2: %s", out)
252 }
253 out, _, _ = inst.ssh(t, aliceKey, "", "mr", "show", "alice/app", "1", "--json")
254 if !strings.Contains(out, "add feature") || !strings.Contains(out, `"state":"merged"`) ||
255 !strings.Contains(out, "imported pull request "+host+"/octo/legacy#2") ||
256 !strings.Contains(out, "nice patch") {
257 t.Fatalf("mr 1: %s", out)
258 }
259 // Re-running imports nothing new.
260 out, _, code = inst.ssh(t, aliceKey, "", "repo", "import-issues", "alice/app",
261 "--from", "octo/legacy", "--api-base", fj.URL+"/api/v1")
262 if code != 0 || !strings.Contains(out, "imported 0 issues, 0 merge requests, 0 comments (3 items already imported)") {
263 t.Fatalf("re-run: %s", out)
264 }
265}
internal/control/ghimport.go +65 −19
@@ -23,8 +23,8 @@ import (
23 23
24func init() { 24func init() {
25 register(Command{Path: []string{"repo", "import-issues"}, 25 register(Command{Path: []string{"repo", "import-issues"},
26 Summary: "import GitHub issue and PR history", 26 Summary: "import issue and PR history from GitHub or Forgejo",
27 Usage: "repo import-issues <owner/name> --from <ghowner/ghrepo> [--token-stdin] [--api-base <url>]", 27 Usage: "repo import-issues <owner/name> --from <owner/repo> [--token-stdin] [--api-base <url>]",
28 ReadsStdin: true, Run: runImportIssues}) 28 ReadsStdin: true, Run: runImportIssues})
29} 29}
30 30
@@ -66,6 +66,45 @@ type ghClient struct {
66 base string 66 base string
67 token string 67 token string
68 http *http.Client 68 http *http.Client
69 // forgejo is set when the API base answers /version, which GitHub
70 // does not. Forgejo (and Gitea, and so Codeberg) mirror GitHub's
71 // issue, pull and comment shapes but not its query parameters:
72 // pages are sized by `limit`, order is `sort=oldest`, and the
73 // comments endpoint has no pages at all — it ignores `page` and
74 // returns everything every time, which paged the old loop forever.
75 forgejo bool
76}
77
78func (g *ghClient) detect() {
79 var v struct{ Version string }
80 g.forgejo = g.get("/version", &v) == nil && v.Version != ""
81}
82
83// issuesQuery lists every issue and pull request oldest first, so local
84// numbers come out in the source's order.
85func (g *ghClient) issuesQuery(from string, page int) string {
86 if g.forgejo {
87 return fmt.Sprintf("/repos/%s/issues?state=all&sort=oldest&limit=50&page=%d", from, page)
88 }
89 return fmt.Sprintf("/repos/%s/issues?state=all&sort=created&direction=asc&per_page=100&page=%d", from, page)
90}
91
92// siteFromAPI turns an API base into the site that serves git and the
93// name attribution carries: api.github.com is github.com, a GitHub
94// Enterprise or Forgejo base drops its /api/vN suffix.
95func siteFromAPI(apiBase string) string {
96 u, err := url.Parse(apiBase)
97 if err != nil {
98 return apiBase
99 }
100 if u.Host == "api.github.com" {
101 u.Host = "github.com"
102 u.Path = ""
103 }
104 u.Path = strings.TrimSuffix(strings.TrimSuffix(u.Path, "/"), "/api/v1")
105 u.Path = strings.TrimSuffix(u.Path, "/api/v3")
106 u.RawQuery, u.Fragment = "", ""
107 return u.String()
69} 108}
70 109
71func (g *ghClient) get(path string, out any) error { 110func (g *ghClient) get(path string, out any) error {
@@ -110,12 +149,7 @@ func runImportIssues(c *Ctx, args []string) int {
110 } 149 }
111 path, from, apiBase, tokenStdin := f.pos(0), f.Value("--from"), f.Value("--api-base"), f.Has("--token-stdin") 150 path, from, apiBase, tokenStdin := f.pos(0), f.Value("--from"), f.Value("--api-base"), f.Has("--token-stdin")
112 if path == "" || from == "" { 151 if path == "" || from == "" {
113 return c.fail(protocol.ExitUsage, "usage: repo import-issues <owner/name> --from <ghowner/ghrepo> [--token-stdin]") 152 return c.fail(protocol.ExitUsage, "usage: repo import-issues <owner/name> --from <owner/repo> [--token-stdin] [--api-base <url>]")
114 }
115 // Accept a bare owner/repo or a full github.com URL.
116 from = strings.TrimSuffix(strings.TrimPrefix(strings.TrimPrefix(from, "https://"), "github.com/"), ".git")
117 if parts := strings.Split(from, "/"); len(parts) != 2 || parts[0] == "" || parts[1] == "" {
118 return c.fail(protocol.ExitUsage, "--from must be <ghowner>/<ghrepo> (or the github.com URL)")
119 } 153 }
120 if apiBase == "" { 154 if apiBase == "" {
121 apiBase = "https://api.github.com" 155 apiBase = "https://api.github.com"
@@ -124,6 +158,15 @@ func runImportIssues(c *Ctx, args []string) int {
124 // webhook target; same rules apply. 158 // webhook target; same rules apply.
125 return c.fail(protocol.ExitUsage, "--api-base: %v", err) 159 return c.fail(protocol.ExitUsage, "--api-base: %v", err)
126 } 160 }
161 site := siteFromAPI(apiBase)
162 host := strings.TrimPrefix(strings.TrimPrefix(site, "https://"), "http://")
163 // Accept a bare owner/repo or the repository's URL on that site.
164 from = strings.TrimPrefix(from, site+"/")
165 from = strings.TrimPrefix(from, host+"/")
166 from = strings.TrimSuffix(from, ".git")
167 if parts := strings.Split(from, "/"); len(parts) != 2 || parts[0] == "" || parts[1] == "" {
168 return c.fail(protocol.ExitUsage, "--from must be <owner>/<repo> (or the repository URL on %s)", site)
169 }
127 repo, code := resolveRepo(c, path, policy.CanWrite) 170 repo, code := resolveRepo(c, path, policy.CanWrite)
128 if code >= 0 { 171 if code >= 0 {
129 return code 172 return code
@@ -142,6 +185,7 @@ func runImportIssues(c *Ctx, args []string) int {
142 token = strings.TrimSpace(line) 185 token = strings.TrimSpace(line)
143 } 186 }
144 g := &ghClient{base: apiBase, token: token, http: &http.Client{Timeout: 30 * time.Second}} 187 g := &ghClient{base: apiBase, token: token, http: &http.Client{Timeout: 30 * time.Second}}
188 g.detect()
145 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name) 189 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
146 190
147 // Pull heads first, so every merge request below has objects to point 191 // Pull heads first, so every merge request below has objects to point
@@ -151,14 +195,13 @@ func runImportIssues(c *Ctx, args []string) int {
151 // import of issues from a repository whose git data is not here yet 195 // import of issues from a repository whose git data is not here yet
152 // is a legitimate thing to do, and the merge requests still arrive 196 // is a legitimate thing to do, and the merge requests still arrive
153 // with their head SHA recorded. 197 // with their head SHA recorded.
154 fetchedPullHeads := fetchPullHeads(c, dir, from, token) 198 fetchedPullHeads := fetchPullHeads(c, dir, site+"/"+from+".git", token)
155 src := "github.com/" + from 199 src := host + "/" + from
156 200
157 var issues, mrs, comments, skipped, headed int 201 var issues, mrs, comments, skipped, headed int
158 for page := 1; ; page++ { 202 for page := 1; ; page++ {
159 var items []ghIssue 203 var items []ghIssue
160 q := fmt.Sprintf("/repos/%s/issues?state=all&sort=created&direction=asc&per_page=100&page=%d", from, page) 204 if err := g.get(g.issuesQuery(from, page), &items); err != nil {
161 if err := g.get(q, &items); err != nil {
162 return c.fail(protocol.ExitFailure, "%v", err) 205 return c.fail(protocol.ExitFailure, "%v", err)
163 } 206 }
164 if len(items) == 0 { 207 if len(items) == 0 {
@@ -262,6 +305,10 @@ func importComments(c *Ctx, g *ghClient, repo store.Repo, from, src string, ghN,
262 } 305 }
263 imported := 0 306 imported := 0
264 for page := 1; ; page++ { 307 for page := 1; ; page++ {
308 // Forgejo returns every comment in one unpaged reply.
309 if g.forgejo && page > 1 {
310 return imported, nil
311 }
265 var cs []ghComment 312 var cs []ghComment
266 q := fmt.Sprintf("/repos/%s/issues/%d/comments?per_page=100&page=%d", url.PathEscape(from), ghN, page) 313 q := fmt.Sprintf("/repos/%s/issues/%d/comments?per_page=100&page=%d", url.PathEscape(from), ghN, page)
267 q = strings.ReplaceAll(q, "%2F", "/") 314 q = strings.ReplaceAll(q, "%2F", "/")
@@ -304,11 +351,11 @@ case "$1" in
304esac 351esac
305` 352`
306 353
307// fetchPullHeads brings refs/pull/*/head into refs/gh-pull/*. Reports 354// fetchPullHeads brings refs/pull/*/head into refs/gh-pull/*; GitHub and
308// whether it worked; a failure is not fatal, since importing issues from 355// Forgejo both publish pull heads under that name. Reports whether it
309// a repository whose git data is not here yet is a reasonable thing to 356// worked; a failure is not fatal, since importing issues from a
310// do. 357// repository whose git data is not here yet is a reasonable thing to do.
311func fetchPullHeads(c *Ctx, dir, from, token string) bool { 358func fetchPullHeads(c *Ctx, dir, remote, token string) bool {
312 env := []string{"GIT_TERMINAL_PROMPT=0", "HOME=" + c.Cfg.Server.Root} 359 env := []string{"GIT_TERMINAL_PROMPT=0", "HOME=" + c.Cfg.Server.Root}
313 if token != "" { 360 if token != "" {
314 askpass := filepath.Join(c.Cfg.Server.Root, "gh-import-askpass.sh") 361 askpass := filepath.Join(c.Cfg.Server.Root, "gh-import-askpass.sh")
@@ -319,8 +366,7 @@ func fetchPullHeads(c *Ctx, dir, from, token string) bool {
319 } 366 }
320 ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) 367 ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
321 defer cancel() 368 defer cancel()
322 url := "https://github.com/" + from + ".git" 369 if err := gitutil.FetchPullHeads(ctx, dir, remote, io.Discard, env); err != nil {
323 if err := gitutil.FetchPullHeads(ctx, dir, url, io.Discard, env); err != nil {
324 return false 370 return false
325 } 371 }
326 return true 372 return true