krz/gitbay

A CLI-first git forge. cli forge git self-hosted https://gitbay.org

Commit bc93d6cbca

bc93d6cbca7bcd029f6cbc63fc599f4670560941

parent: 37a776060e

Verified · cmc ci/build: success ci/test: success

cmc <hello@cleberg.net> · 2026-09-08 05:44 UTC 

control, e2e, wiki: import-issues reads Forgejo through --api-base

A live run against codeberg.org/oneprog/ada-judge found three breaks.
The comments endpoint has no pages and ignores `page`, so the comment
loop never ended; the issue list ignores `per_page` and
`sort=created&direction=asc`, so items arrived newest first and local
numbers came out reversed; attribution and the pull-head fetch named
github.com whatever the API base was.

The client probes /version once, which Forgejo answers and GitHub does
not. On a Forgejo base it pages with `limit`, orders with
`sort=oldest`, and reads comments once. The site is derived from the
API base for attribution, the pull-head fetch, and a --from given as
the repository's URL. The same run now imports 24 issues, one merged
MR with its head, and 6 comments in the source's order.

Ref #191
.gitbay/wiki/FAQ.org +3 −3
@@ -9,9 +9,9 @@
99 when the one below merges, and refuses squash or rebase under a stack.
1010 No stack command; the workflow is branches. See
1111 [[Stacked-MRs][Stacked merge requests]].
12- Moving from GitHub? :: =gitbay repo import= (git data) then
13 =gitbay repo import-issues= (history); =repo mirror= covers the
14 transition window.
12- Moving from GitHub or Codeberg? :: =gitbay repo import= (git data)
13 then =gitbay repo import-issues= (history, from GitHub or any Forgejo
14 =--api-base=); =repo mirror= covers the transition window.
1515- What is not planned? :: A container or package registry (scope creep
1616 away from "forge"; external registries integrate through CI), an
1717 email patch flow (revisit only if sourcehut-style demand appears),
.gitbay/wiki/Users.org +10 −5
@@ -207,17 +207,22 @@ gitbay repo archive you/project # read-only: pushes and issu
207207gitbay repo unarchive you/project # writes refused, browsing intact
208208#+end_src
209209
210Import from another forge — git data first, then optionally the GitHub
211issue and PR history (issues keep state/labels/comments; PRs land as
212closed or merged MRs with their discussion; originals are attributed
213inline since foreign authors have no local account; re-running resumes
214where it stopped):
210Import from another forge — git data first, then optionally the issue
211and PR history from GitHub or any Forgejo instance such as Codeberg
212(issues keep state/labels/comments; PRs land as closed or merged MRs
213with their discussion; originals are attributed inline since foreign
214authors have no local account; re-running resumes where it stopped):
215215#+begin_src sh
216216gitbay repo import you/mirror --from https://github.com/you/repo.git \
217217 [--private] [--token-stdin] # token on stdin, never in the URL
218218gitbay repo import-issues you/mirror --from you/repo --token-stdin
219gitbay repo import-issues you/mirror --from you/repo \
220 --api-base https://codeberg.org/api/v1 # Forgejo: the site's /api/v1
219221#+end_src
220222
223Sourcehut has no API of that shape; =repo import= takes its git data
224and the todo.sr.ht tracker is not read.
225
221226Moving between gitbay instances (no lock-in): run on the TARGET, with
222227your key registered on both sides. Profile, repos with settings,
223228issues, MRs, and comments replay with attribution; git data mirrors
e2e/ghimport_test.go +111 −2
@@ -92,6 +92,7 @@ func TestGitHubIssueImport(t *testing.T) {
9292 mustGit(t, dir, env, "push", "-q", "origin", "main")
9393
9494 gh := fakeGitHub(t)
95 host := strings.TrimPrefix(gh.URL, "http://")
9596 out, errOut, code := inst.ssh(t, aliceKey, "sekrit\n", "repo", "import-issues", "alice/app",
9697 "--from", "octo/legacy", "--token-stdin", "--api-base", gh.URL)
9798 if code != 0 {
@@ -105,7 +106,7 @@ func TestGitHubIssueImport(t *testing.T) {
105106 out, _, _ = inst.ssh(t, aliceKey, "", "issue", "show", "alice/app", "1", "--json")
106107 if !strings.Contains(out, "old bug") || !strings.Contains(out, `"state":"closed"`) ||
107108 !strings.Contains(out, `"labels":["bug"]`) ||
108 !strings.Contains(out, "imported issue github.com/octo/legacy#1") ||
109 !strings.Contains(out, "imported issue "+host+"/octo/legacy#1") ||
109110 !strings.Contains(out, "@octofan, 2019-03-04") {
110111 t.Fatalf("issue 1: %s", out)
111112 }
@@ -118,7 +119,7 @@ func TestGitHubIssueImport(t *testing.T) {
118119 // MR !1 (GitHub PR #2): merged, discussion imported.
119120 out, _, _ = inst.ssh(t, aliceKey, "", "mr", "show", "alice/app", "1", "--json")
120121 if !strings.Contains(out, "add feature") || !strings.Contains(out, `"state":"merged"`) ||
121 !strings.Contains(out, "imported pull request github.com/octo/legacy#2") ||
122 !strings.Contains(out, "imported pull request "+host+"/octo/legacy#2") ||
122123 !strings.Contains(out, "nice patch") {
123124 t.Fatalf("mr 1: %s", out)
124125 }
@@ -154,3 +155,111 @@ func TestGitHubImportSSRFGuard(t *testing.T) {
154155 t.Fatalf("local api-base allowed: exit %d, %s", code, errOut)
155156 }
156157}
158
159// fakeForgejo serves the Forgejo shape of the same API under /api/v1:
160// GitHub's issue, pull and comment objects, but pages sized by `limit`,
161// order by `sort=oldest`, a /version endpoint, and a comments endpoint
162// that ignores `page` and returns everything every time.
163func fakeForgejo(t *testing.T) *httptest.Server {
164 t.Helper()
165 mux := http.NewServeMux()
166 mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
167 fmt.Fprint(w, `{"version":"9.0.0+gitea-1.22.0"}`)
168 })
169 mux.HandleFunc("/api/v1/repos/octo/legacy/issues", func(w http.ResponseWriter, r *http.Request) {
170 q := r.URL.Query()
171 if q.Get("page") != "1" {
172 fmt.Fprint(w, "[]")
173 return
174 }
175 items := []string{
176 `{"number":1,"title":"old bug","body":"it crashed","state":"closed",
177 "created_at":"2019-03-04T10:00:00+01:00","user":{"login":"octofan"},
178 "labels":[{"name":"bug"}],"comments":0}`,
179 `{"number":2,"title":"add feature","body":"the patch","state":"closed",
180 "created_at":"2020-06-01T10:00:00Z","user":{"login":"drive-by"},
181 "labels":[],"comments":1,"pull_request":{"merged":true}}`,
182 `{"number":3,"title":"still open","body":"discuss","state":"open",
183 "created_at":"2021-01-01T10:00:00Z","user":{"login":"octofan"},
184 "labels":[],"comments":2}`,
185 }
186 // Forgejo's default is newest first; only sort=oldest gives
187 // the order local numbering depends on.
188 if q.Get("sort") != "oldest" || q.Get("limit") == "" {
189 items[0], items[2] = items[2], items[0]
190 }
191 fmt.Fprint(w, "["+strings.Join(items, ",")+"]")
192 })
193 mux.HandleFunc("/api/v1/repos/octo/legacy/pulls/2", func(w http.ResponseWriter, r *http.Request) {
194 fmt.Fprint(w, `{"merged_at":"2020-06-02T10:00:00Z",
195 "head":{"sha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","ref":"feature"},
196 "base":{"sha":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","ref":"main"}}`)
197 })
198 comments := func(payload string) http.HandlerFunc {
199 return func(w http.ResponseWriter, r *http.Request) {
200 // No paging on this endpoint: the real one ignores `page`
201 // and returns everything, so a caller walking pages never
202 // stops. Answer a second page with an error so the test
203 // fails instead of hanging.
204 if p := r.URL.Query().Get("page"); p != "" && p != "1" {
205 http.Error(w, "unpaged endpoint asked for page "+p, 500)
206 return
207 }
208 fmt.Fprint(w, payload)
209 }
210 }
211 mux.HandleFunc("/api/v1/repos/octo/legacy/issues/2/comments", comments(
212 `[{"id":101,"body":"nice patch","created_at":"2020-06-01T11:00:00Z","user":{"login":"maintainer"}}]`))
213 mux.HandleFunc("/api/v1/repos/octo/legacy/issues/3/comments", comments(
214 `[{"id":102,"body":"me too","created_at":"2021-01-02T10:00:00Z","user":{"login":"other"}},
215 {"id":103,"body":"still happening","created_at":"2021-02-01T10:00:00Z","user":{"login":"octofan"}}]`))
216 srv := httptest.NewServer(mux)
217 t.Cleanup(srv.Close)
218 return srv
219}
220
221func TestForgejoIssueImport(t *testing.T) {
222 inst := startInstanceWith(t, "[webhooks]\nallow_local = true\n")
223 aliceKey := inst.newKey(t, "alice")
224 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
225 if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
226 t.Fatalf("repo create: %s", errOut)
227 }
228 fj := fakeForgejo(t)
229 host := strings.TrimPrefix(fj.URL, "http://")
230 // --from as the repository's URL on the site, the way a Codeberg
231 // user copies it from the address bar.
232 out, errOut, code := inst.ssh(t, aliceKey, "", "repo", "import-issues", "alice/app",
233 "--from", fj.URL+"/octo/legacy", "--api-base", fj.URL+"/api/v1")
234 if code != 0 {
235 t.Fatalf("import: %s", errOut)
236 }
237 if !strings.Contains(out, "imported 2 issues, 1 merge requests, 3 comments") {
238 t.Fatalf("summary: %s", out)
239 }
240 // Oldest first, attributed to the site the API base belongs to.
241 out, _, _ = inst.ssh(t, aliceKey, "", "issue", "show", "alice/app", "1", "--json")
242 if !strings.Contains(out, "old bug") || !strings.Contains(out, `"state":"closed"`) ||
243 !strings.Contains(out, `"labels":["bug"]`) ||
244 !strings.Contains(out, "imported issue "+host+"/octo/legacy#1") ||
245 !strings.Contains(out, "@octofan, 2019-03-04") {
246 t.Fatalf("issue 1: %s", out)
247 }
248 out, _, _ = inst.ssh(t, aliceKey, "", "issue", "show", "alice/app", "2", "--json")
249 if !strings.Contains(out, "still open") || !strings.Contains(out, "me too") ||
250 !strings.Contains(out, "still happening") {
251 t.Fatalf("issue 2: %s", out)
252 }
253 out, _, _ = inst.ssh(t, aliceKey, "", "mr", "show", "alice/app", "1", "--json")
254 if !strings.Contains(out, "add feature") || !strings.Contains(out, `"state":"merged"`) ||
255 !strings.Contains(out, "imported pull request "+host+"/octo/legacy#2") ||
256 !strings.Contains(out, "nice patch") {
257 t.Fatalf("mr 1: %s", out)
258 }
259 // Re-running imports nothing new.
260 out, _, code = inst.ssh(t, aliceKey, "", "repo", "import-issues", "alice/app",
261 "--from", "octo/legacy", "--api-base", fj.URL+"/api/v1")
262 if code != 0 || !strings.Contains(out, "imported 0 issues, 0 merge requests, 0 comments (3 items already imported)") {
263 t.Fatalf("re-run: %s", out)
264 }
265}
internal/control/ghimport.go +65 −19
@@ -23,8 +23,8 @@ import (
2323
2424func init() {
2525 register(Command{Path: []string{"repo", "import-issues"},
26 Summary: "import GitHub issue and PR history",
27 Usage: "repo import-issues <owner/name> --from <ghowner/ghrepo> [--token-stdin] [--api-base <url>]",
26 Summary: "import issue and PR history from GitHub or Forgejo",
27 Usage: "repo import-issues <owner/name> --from <owner/repo> [--token-stdin] [--api-base <url>]",
2828 ReadsStdin: true, Run: runImportIssues})
2929}
3030
@@ -66,6 +66,45 @@ type ghClient struct {
6666 base string
6767 token string
6868 http *http.Client
69 // forgejo is set when the API base answers /version, which GitHub
70 // does not. Forgejo (and Gitea, and so Codeberg) mirror GitHub's
71 // issue, pull and comment shapes but not its query parameters:
72 // pages are sized by `limit`, order is `sort=oldest`, and the
73 // comments endpoint has no pages at all — it ignores `page` and
74 // returns everything every time, which paged the old loop forever.
75 forgejo bool
76}
77
78func (g *ghClient) detect() {
79 var v struct{ Version string }
80 g.forgejo = g.get("/version", &v) == nil && v.Version != ""
81}
82
83// issuesQuery lists every issue and pull request oldest first, so local
84// numbers come out in the source's order.
85func (g *ghClient) issuesQuery(from string, page int) string {
86 if g.forgejo {
87 return fmt.Sprintf("/repos/%s/issues?state=all&sort=oldest&limit=50&page=%d", from, page)
88 }
89 return fmt.Sprintf("/repos/%s/issues?state=all&sort=created&direction=asc&per_page=100&page=%d", from, page)
90}
91
92// siteFromAPI turns an API base into the site that serves git and the
93// name attribution carries: api.github.com is github.com, a GitHub
94// Enterprise or Forgejo base drops its /api/vN suffix.
95func siteFromAPI(apiBase string) string {
96 u, err := url.Parse(apiBase)
97 if err != nil {
98 return apiBase
99 }
100 if u.Host == "api.github.com" {
101 u.Host = "github.com"
102 u.Path = ""
103 }
104 u.Path = strings.TrimSuffix(strings.TrimSuffix(u.Path, "/"), "/api/v1")
105 u.Path = strings.TrimSuffix(u.Path, "/api/v3")
106 u.RawQuery, u.Fragment = "", ""
107 return u.String()
69108}
70109
71110func (g *ghClient) get(path string, out any) error {
@@ -110,12 +149,7 @@ func runImportIssues(c *Ctx, args []string) int {
110149 }
111150 path, from, apiBase, tokenStdin := f.pos(0), f.Value("--from"), f.Value("--api-base"), f.Has("--token-stdin")
112151 if path == "" || from == "" {
113 return c.fail(protocol.ExitUsage, "usage: repo import-issues <owner/name> --from <ghowner/ghrepo> [--token-stdin]")
114 }
115 // Accept a bare owner/repo or a full github.com URL.
116 from = strings.TrimSuffix(strings.TrimPrefix(strings.TrimPrefix(from, "https://"), "github.com/"), ".git")
117 if parts := strings.Split(from, "/"); len(parts) != 2 || parts[0] == "" || parts[1] == "" {
118 return c.fail(protocol.ExitUsage, "--from must be <ghowner>/<ghrepo> (or the github.com URL)")
152 return c.fail(protocol.ExitUsage, "usage: repo import-issues <owner/name> --from <owner/repo> [--token-stdin] [--api-base <url>]")
119153 }
120154 if apiBase == "" {
121155 apiBase = "https://api.github.com"
@@ -124,6 +158,15 @@ func runImportIssues(c *Ctx, args []string) int {
124158 // webhook target; same rules apply.
125159 return c.fail(protocol.ExitUsage, "--api-base: %v", err)
126160 }
161 site := siteFromAPI(apiBase)
162 host := strings.TrimPrefix(strings.TrimPrefix(site, "https://"), "http://")
163 // Accept a bare owner/repo or the repository's URL on that site.
164 from = strings.TrimPrefix(from, site+"/")
165 from = strings.TrimPrefix(from, host+"/")
166 from = strings.TrimSuffix(from, ".git")
167 if parts := strings.Split(from, "/"); len(parts) != 2 || parts[0] == "" || parts[1] == "" {
168 return c.fail(protocol.ExitUsage, "--from must be <owner>/<repo> (or the repository URL on %s)", site)
169 }
127170 repo, code := resolveRepo(c, path, policy.CanWrite)
128171 if code >= 0 {
129172 return code
@@ -142,6 +185,7 @@ func runImportIssues(c *Ctx, args []string) int {
142185 token = strings.TrimSpace(line)
143186 }
144187 g := &ghClient{base: apiBase, token: token, http: &http.Client{Timeout: 30 * time.Second}}
188 g.detect()
145189 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
146190
147191 // Pull heads first, so every merge request below has objects to point
@@ -151,14 +195,13 @@ func runImportIssues(c *Ctx, args []string) int {
151195 // import of issues from a repository whose git data is not here yet
152196 // is a legitimate thing to do, and the merge requests still arrive
153197 // with their head SHA recorded.
154 fetchedPullHeads := fetchPullHeads(c, dir, from, token)
155 src := "github.com/" + from
198 fetchedPullHeads := fetchPullHeads(c, dir, site+"/"+from+".git", token)
199 src := host + "/" + from
156200
157201 var issues, mrs, comments, skipped, headed int
158202 for page := 1; ; page++ {
159203 var items []ghIssue
160 q := fmt.Sprintf("/repos/%s/issues?state=all&sort=created&direction=asc&per_page=100&page=%d", from, page)
161 if err := g.get(q, &items); err != nil {
204 if err := g.get(g.issuesQuery(from, page), &items); err != nil {
162205 return c.fail(protocol.ExitFailure, "%v", err)
163206 }
164207 if len(items) == 0 {
@@ -262,6 +305,10 @@ func importComments(c *Ctx, g *ghClient, repo store.Repo, from, src string, ghN,
262305 }
263306 imported := 0
264307 for page := 1; ; page++ {
308 // Forgejo returns every comment in one unpaged reply.
309 if g.forgejo && page > 1 {
310 return imported, nil
311 }
265312 var cs []ghComment
266313 q := fmt.Sprintf("/repos/%s/issues/%d/comments?per_page=100&page=%d", url.PathEscape(from), ghN, page)
267314 q = strings.ReplaceAll(q, "%2F", "/")
@@ -304,11 +351,11 @@ case "$1" in
304351esac
305352`
306353
307// fetchPullHeads brings refs/pull/*/head into refs/gh-pull/*. Reports
308// whether it worked; a failure is not fatal, since importing issues from
309// a repository whose git data is not here yet is a reasonable thing to
310// do.
311func fetchPullHeads(c *Ctx, dir, from, token string) bool {
354// fetchPullHeads brings refs/pull/*/head into refs/gh-pull/*; GitHub and
355// Forgejo both publish pull heads under that name. Reports whether it
356// worked; a failure is not fatal, since importing issues from a
357// repository whose git data is not here yet is a reasonable thing to do.
358func fetchPullHeads(c *Ctx, dir, remote, token string) bool {
312359 env := []string{"GIT_TERMINAL_PROMPT=0", "HOME=" + c.Cfg.Server.Root}
313360 if token != "" {
314361 askpass := filepath.Join(c.Cfg.Server.Root, "gh-import-askpass.sh")
@@ -319,8 +366,7 @@ func fetchPullHeads(c *Ctx, dir, from, token string) bool {
319366 }
320367 ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
321368 defer cancel()
322 url := "https://github.com/" + from + ".git"
323 if err := gitutil.FetchPullHeads(ctx, dir, url, io.Discard, env); err != nil {
369 if err := gitutil.FetchPullHeads(ctx, dir, remote, io.Discard, env); err != nil {
324370 return false
325371 }
326372 return true