krz/gitbay

A CLI-first git forge.

cli forge git self-hosted

https://gitbay.org

Commit 4b19481915

4b1948191569eba678216f0734c99cebd219c6c8

parent: 1ca9f7fdd3

Verified · cmc

cmc <hello@cleberg.net> · 2026-08-24T21:09:31Z 

Import GitHub issue and PR history

Closes #17

repo import-issues <owner/name> --from <ghowner/ghrepo> walks the
GitHub REST API (token via stdin, never argv or stored; --api-base for
GHE, validated with the webhook SSRF rules): issues arrive with state,
labels, and comments; PRs become merged/closed MRs with their
discussion, merged_base from the PR base, and refs/merge-requests/N/head
pointed at the PR head when the mirror already holds the objects.
Originals are attributed inline (author + date) since foreign users
have no local account. import_markers (migration 0016) make the run
resumable at item and comment granularity — re-running imports nothing
twice. Review-diff comments are not imported.
cmd/gitbay/main.go +5
@@ -138,6 +138,9 @@ func usesStdin(args []string) bool {
138138 if a == "--file" && i+1 < len(args) && args[i+1] == "-" {
139139 return true
140140 }
141 if a == "--token-stdin" {
142 return true
143 }
141144 }
142145 return false
143146 }
@@ -233,6 +236,8 @@ func repoCmd() *cobra.Command {
233236 pass("unarchive", "unarchive a repository", passOpts{server: []string{"repo", "unarchive"}, needsRepo: true}),
234237 local("clone", "clone via ssh: gitbay repo clone <owner/name> [dir]", cmdRepoClone),
235238 importCmd(),
239 pass("import-issues", "import GitHub issue/PR history: --from <ghowner/ghrepo> [--token-stdin]",
240 passOpts{server: []string{"repo", "import-issues"}, needsRepo: true, stdinOK: true}),
236241 group("deploy-key", "repository-bound CI keys",
237242 pass("add", "bind a key: [--rw] < key.pub", passOpts{server: []string{"repo", "deploy-key", "add"}, needsRepo: true, stdinOK: true}),
238243 pass("list", "list deploy keys", passOpts{server: []string{"repo", "deploy-key", "list"}, needsRepo: true}),
docs/users.org +6 −2
@@ -116,11 +116,15 @@ gitbay repo archive you/project # read-only: pushes and issu
116116 gitbay repo unarchive you/project # writes refused, browsing intact
117117 #+end_src
118118
119Import from another forge (git data only — issues and PRs do not
120transfer):
119Import from another forge — git data first, then optionally the GitHub
120issue and PR history (issues keep state/labels/comments; PRs land as
121closed or merged MRs with their discussion; originals are attributed
122inline since foreign authors have no local account; re-running resumes
123where it stopped):
121124 #+begin_src sh
122125 gitbay repo import you/mirror --from https://github.com/you/repo.git \
123126 [--private] [--token-stdin] # token on stdin, never in the URL
127gitbay repo import-issues you/mirror --from you/repo --token-stdin
124128 #+end_src
125129
126130 * Organizations
e2e/ghimport_test.go added +156
@@ -0,0 +1,156 @@
1package e2e
2
3import (
4 "fmt"
5 "net/http"
6 "net/http/httptest"
7 "os"
8 "path/filepath"
9 "strings"
10 "testing"
11)
12
13// fakeGitHub serves just enough of the GitHub REST API for the importer.
14func fakeGitHub(t *testing.T) *httptest.Server {
15 t.Helper()
16 mux := http.NewServeMux()
17 auth := func(w http.ResponseWriter, r *http.Request) bool {
18 if r.Header.Get("Authorization") != "Bearer sekrit" {
19 w.WriteHeader(401)
20 return false
21 }
22 return true
23 }
24 mux.HandleFunc("/repos/octo/legacy/issues", func(w http.ResponseWriter, r *http.Request) {
25 if !auth(w, r) {
26 return
27 }
28 if r.URL.Query().Get("page") != "1" {
29 fmt.Fprint(w, "[]")
30 return
31 }
32 fmt.Fprint(w, `[
33 {"number":1,"title":"old bug","body":"it crashed","state":"closed",
34 "created_at":"2019-03-04T10:00:00Z","user":{"login":"octofan"},
35 "labels":[{"name":"bug"}],"comments":0},
36 {"number":2,"title":"add feature","body":"the patch","state":"closed",
37 "created_at":"2020-06-01T10:00:00Z","user":{"login":"drive-by"},
38 "labels":[],"comments":1,"pull_request":{}},
39 {"number":3,"title":"still open","body":"discuss","state":"open",
40 "created_at":"2021-01-01T10:00:00Z","user":{"login":"octofan"},
41 "labels":[],"comments":2}
42 ]`)
43 })
44 mux.HandleFunc("/repos/octo/legacy/pulls/2", func(w http.ResponseWriter, r *http.Request) {
45 if !auth(w, r) {
46 return
47 }
48 fmt.Fprint(w, `{"merged_at":"2020-06-02T10:00:00Z",
49 "head":{"sha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","ref":"feature"},
50 "base":{"sha":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","ref":"main"}}`)
51 })
52 comments := func(payload string) http.HandlerFunc {
53 return func(w http.ResponseWriter, r *http.Request) {
54 if !auth(w, r) {
55 return
56 }
57 if r.URL.Query().Get("page") != "1" {
58 fmt.Fprint(w, "[]")
59 return
60 }
61 fmt.Fprint(w, payload)
62 }
63 }
64 mux.HandleFunc("/repos/octo/legacy/issues/2/comments", comments(
65 `[{"id":101,"body":"nice patch","created_at":"2020-06-01T11:00:00Z","user":{"login":"maintainer"}}]`))
66 mux.HandleFunc("/repos/octo/legacy/issues/3/comments", comments(
67 `[{"id":102,"body":"me too","created_at":"2021-01-02T10:00:00Z","user":{"login":"other"}},
68 {"id":103,"body":"still happening","created_at":"2021-02-01T10:00:00Z","user":{"login":"octofan"}}]`))
69 srv := httptest.NewServer(mux)
70 t.Cleanup(srv.Close)
71 return srv
72}
73
74func TestGitHubIssueImport(t *testing.T) {
75 // allow_local lets --api-base reach the loopback fake; a default
76 // instance refuses it (see the SSRF check at the end).
77 inst := startInstanceWith(t, "[webhooks]\nallow_local = true\n")
78 aliceKey := inst.newKey(t, "alice")
79 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
80
81 if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
82 t.Fatalf("repo create: %s", errOut)
83 }
84 work := t.TempDir()
85 env := inst.gitEnv(aliceKey)
86 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
87 dir := filepath.Join(work, "w")
88 os.WriteFile(filepath.Join(dir, "a.txt"), []byte("a\n"), 0o644)
89 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
90 mustGit(t, dir, env, "add", ".")
91 mustGit(t, dir, env, "commit", "-q", "-m", "base")
92 mustGit(t, dir, env, "push", "-q", "origin", "main")
93
94 gh := fakeGitHub(t)
95 out, errOut, code := inst.ssh(t, aliceKey, "sekrit\n", "repo", "import-issues", "alice/app",
96 "--from", "octo/legacy", "--token-stdin", "--api-base", gh.URL)
97 if code != 0 {
98 t.Fatalf("import: %s", errOut)
99 }
100 if !strings.Contains(out, "imported 2 issues, 1 merge requests, 3 comments") {
101 t.Fatalf("summary: %s", out)
102 }
103
104 // Issue #1 (GitHub #1): closed, labeled, attributed.
105 out, _, _ = inst.ssh(t, aliceKey, "", "issue", "show", "alice/app", "1", "--json")
106 if !strings.Contains(out, "old bug") || !strings.Contains(out, `"state":"closed"`) ||
107 !strings.Contains(out, `"labels":["bug"]`) ||
108 !strings.Contains(out, "imported issue github.com/octo/legacy#1") ||
109 !strings.Contains(out, "@octofan, 2019-03-04") {
110 t.Fatalf("issue 1: %s", out)
111 }
112 // Issue #2 (GitHub #3): open, two attributed comments.
113 out, _, _ = inst.ssh(t, aliceKey, "", "issue", "show", "alice/app", "2", "--json")
114 if !strings.Contains(out, "still open") || !strings.Contains(out, `"state":"open"`) ||
115 !strings.Contains(out, "me too") || !strings.Contains(out, "@other, 2021-01-02") {
116 t.Fatalf("issue 2: %s", out)
117 }
118 // MR !1 (GitHub PR #2): merged, discussion imported.
119 out, _, _ = inst.ssh(t, aliceKey, "", "mr", "show", "alice/app", "1", "--json")
120 if !strings.Contains(out, "add feature") || !strings.Contains(out, `"state":"merged"`) ||
121 !strings.Contains(out, "imported pull request github.com/octo/legacy#2") ||
122 !strings.Contains(out, "nice patch") {
123 t.Fatalf("mr 1: %s", out)
124 }
125
126 // Re-running imports nothing new — fully resumable.
127 out, _, code = inst.ssh(t, aliceKey, "sekrit\n", "repo", "import-issues", "alice/app",
128 "--from", "octo/legacy", "--token-stdin", "--api-base", gh.URL)
129 if code != 0 || !strings.Contains(out, "imported 0 issues, 0 merge requests, 0 comments (3 items already imported)") {
130 t.Fatalf("re-run: %s", out)
131 }
132 out, _, _ = inst.ssh(t, aliceKey, "", "issue", "list", "alice/app", "--state", "all")
133 if strings.Count(out, "\n") != 2 {
134 t.Fatalf("issues duplicated:\n%s", out)
135 }
136
137 // A wrong token surfaces the API error.
138 if _, errOut, code := inst.ssh(t, aliceKey, "wrong\n", "repo", "import-issues", "alice/app",
139 "--from", "octo/legacy", "--token-stdin", "--api-base", gh.URL); code == 0 || !strings.Contains(errOut, "401") {
140 t.Fatalf("bad token: exit %d, %s", code, errOut)
141 }
142}
143
144func TestGitHubImportSSRFGuard(t *testing.T) {
145 inst := startInstance(t) // allow_local off: default posture
146 aliceKey := inst.newKey(t, "alice")
147 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
148 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
149 t.Fatal("repo create failed")
150 }
151 _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "import-issues", "alice/app",
152 "--from", "octo/legacy", "--api-base", "http://127.0.0.1:9999")
153 if code != 2 || !strings.Contains(errOut, "SSRF") {
154 t.Fatalf("local api-base allowed: exit %d, %s", code, errOut)
155 }
156}
internal/control/ghimport.go added +290
@@ -0,0 +1,290 @@
1package control
2
3import (
4 "bufio"
5 "encoding/json"
6 "fmt"
7 "io"
8 "net/http"
9 "net/url"
10 "strconv"
11 "strings"
12 "time"
13
14 "gitbay.org/gitbay/internal/gitutil"
15 "gitbay.org/gitbay/internal/policy"
16 "gitbay.org/gitbay/internal/protocol"
17 "gitbay.org/gitbay/internal/store"
18 "gitbay.org/gitbay/internal/webhook"
19)
20
21func init() {
22 register(Command{Path: []string{"repo", "import-issues"},
23 Summary: "import GitHub issue and PR history: repo import-issues <owner/name> --from <ghowner/ghrepo> [--token-stdin] [--api-base <url>]",
24 ReadsStdin: true, Run: runImportIssues})
25}
26
27// GitHub API shapes, minimal.
28type ghUser struct {
29 Login string `json:"login"`
30}
31type ghIssue struct {
32 Number int64 `json:"number"`
33 Title string `json:"title"`
34 Body string `json:"body"`
35 State string `json:"state"`
36 CreatedAt string `json:"created_at"`
37 User ghUser `json:"user"`
38 Labels []struct{ Name string } `json:"labels"`
39 PullRequest *struct{} `json:"pull_request"`
40 Comments int `json:"comments"`
41}
42type ghPull struct {
43 MergedAt string `json:"merged_at"`
44 Head struct {
45 SHA string `json:"sha"`
46 Ref string `json:"ref"`
47 } `json:"head"`
48 Base struct {
49 SHA string `json:"sha"`
50 Ref string `json:"ref"`
51 } `json:"base"`
52}
53type ghComment struct {
54 ID int64 `json:"id"`
55 Body string `json:"body"`
56 CreatedAt string `json:"created_at"`
57 User ghUser `json:"user"`
58}
59
60type ghClient struct {
61 base string
62 token string
63 http *http.Client
64}
65
66func (g *ghClient) get(path string, out any) error {
67 req, err := http.NewRequest("GET", g.base+path, nil)
68 if err != nil {
69 return err
70 }
71 req.Header.Set("Accept", "application/vnd.github+json")
72 if g.token != "" {
73 req.Header.Set("Authorization", "Bearer "+g.token)
74 }
75 resp, err := g.http.Do(req)
76 if err != nil {
77 return err
78 }
79 defer resp.Body.Close()
80 if resp.StatusCode != 200 {
81 body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
82 return fmt.Errorf("GitHub API %s: %s: %.200s", path, resp.Status, body)
83 }
84 return json.NewDecoder(resp.Body).Decode(out)
85}
86
87func ghDate(iso string) string {
88 if t, err := time.Parse(time.RFC3339, iso); err == nil {
89 return t.UTC().Format("2006-01-02")
90 }
91 return iso
92}
93
94// attribution heads every imported body: foreign authors have no local
95// account, so the original author and date live in the text.
96func attribution(src string, n int64, kind, login, date string) string {
97 return fmt.Sprintf("> imported %s %s#%d — @%s, %s\n\n", kind, src, n, login, ghDate(date))
98}
99
100func runImportIssues(c *Ctx, args []string) int {
101 var path, from, apiBase string
102 tokenStdin := false
103 for i := 0; i < len(args); i++ {
104 switch args[i] {
105 case "--from", "--api-base":
106 if i+1 >= len(args) {
107 return c.fail(protocol.ExitUsage, "%s requires a value", args[i])
108 }
109 if args[i] == "--from" {
110 from = args[i+1]
111 } else {
112 apiBase = args[i+1]
113 }
114 i++
115 case "--token-stdin":
116 tokenStdin = true
117 default:
118 if path != "" {
119 return c.fail(protocol.ExitUsage, "unexpected argument %q", args[i])
120 }
121 path = args[i]
122 }
123 }
124 if path == "" || from == "" {
125 return c.fail(protocol.ExitUsage, "usage: repo import-issues <owner/name> --from <ghowner/ghrepo> [--token-stdin]")
126 }
127 // Accept a bare owner/repo or a full github.com URL.
128 from = strings.TrimSuffix(strings.TrimPrefix(strings.TrimPrefix(from, "https://"), "github.com/"), ".git")
129 if parts := strings.Split(from, "/"); len(parts) != 2 || parts[0] == "" || parts[1] == "" {
130 return c.fail(protocol.ExitUsage, "--from must be <ghowner>/<ghrepo> (or the github.com URL)")
131 }
132 if apiBase == "" {
133 apiBase = "https://api.github.com"
134 } else if err := webhook.ValidateURL(apiBase, c.Cfg.Webhooks.AllowLocal); err != nil {
135 // A writer-supplied API base is the same SSRF surface as a
136 // webhook target; same rules apply.
137 return c.fail(protocol.ExitUsage, "--api-base: %v", err)
138 }
139 repo, code := resolveRepo(c, path, policy.CanWrite)
140 if code >= 0 {
141 return code
142 }
143 if code := refuseArchived(c, repo); code >= 0 {
144 return code
145 }
146 token := ""
147 if tokenStdin {
148 // Same discipline as repo import: token on stdin, never argv,
149 // never stored.
150 line, err := bufio.NewReader(c.Stdin).ReadString('\n')
151 if err != nil && line == "" {
152 return c.fail(protocol.ExitUsage, "--token-stdin: no token on stdin")
153 }
154 token = strings.TrimSpace(line)
155 }
156 g := &ghClient{base: apiBase, token: token, http: &http.Client{Timeout: 30 * time.Second}}
157 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
158 src := "github.com/" + from
159
160 var issues, mrs, comments, skipped int
161 for page := 1; ; page++ {
162 var items []ghIssue
163 q := fmt.Sprintf("/repos/%s/issues?state=all&sort=created&direction=asc&per_page=100&page=%d", from, page)
164 if err := g.get(q, &items); err != nil {
165 return c.fail(protocol.ExitFailure, "%v", err)
166 }
167 if len(items) == 0 {
168 break
169 }
170 for _, it := range items {
171 key := fmt.Sprintf("gh:%d", it.Number)
172 val, seen, err := c.Store.ImportMarker(repo.ID, key)
173 if err != nil {
174 return c.fail(protocol.ExitFailure, "%v", err)
175 }
176 var localN int64
177 isPR := it.PullRequest != nil
178 if seen {
179 localN, _ = strconv.ParseInt(strings.TrimPrefix(strings.TrimPrefix(val, "issue:"), "mr:"), 10, 64)
180 skipped++
181 } else if isPR {
182 var pr ghPull
183 if err := g.get(fmt.Sprintf("/repos/%s/pulls/%d", from, it.Number), &pr); err != nil {
184 return c.fail(protocol.ExitFailure, "%v", err)
185 }
186 body := attribution(src, it.Number, "pull request", it.User.Login, it.CreatedAt) + it.Body
187 localN, err = c.Store.CreateMR(repo.ID, c.User.ID, repo.ID, pr.Head.Ref, pr.Base.Ref, it.Title, body, pr.Head.SHA)
188 if err != nil {
189 return c.fail(protocol.ExitFailure, "%v", err)
190 }
191 mr, err := c.Store.MRByNumber(repo.ID, localN)
192 if err != nil {
193 return c.fail(protocol.ExitFailure, "%v", err)
194 }
195 if pr.MergedAt != "" {
196 c.Store.MarkMerged(mr.ID, pr.Base.SHA)
197 } else {
198 c.Store.SetMRState(mr.ID, "closed")
199 }
200 // Point the MR head ref at the PR head when the mirror
201 // already holds the objects (refs/pull backups).
202 if pr.Head.SHA != "" && gitutil.HasCommit(dir, pr.Head.SHA) {
203 gitutil.UpdateRefCAS(dir, fmt.Sprintf("refs/merge-requests/%d/head", localN), pr.Head.SHA, "")
204 }
205 c.Store.SetImportMarker(repo.ID, key, fmt.Sprintf("mr:%d", localN))
206 mrs++
207 } else {
208 body := attribution(src, it.Number, "issue", it.User.Login, it.CreatedAt) + it.Body
209 localN, err = c.Store.CreateIssue(repo.ID, c.User.ID, it.Title, body)
210 if err != nil {
211 return c.fail(protocol.ExitFailure, "%v", err)
212 }
213 iss, err := c.Store.IssueByNumber(repo.ID, localN)
214 if err != nil {
215 return c.fail(protocol.ExitFailure, "%v", err)
216 }
217 for _, l := range it.Labels {
218 c.Store.SetIssueLabel(repo.ID, iss.ID, l.Name, true)
219 }
220 if it.State != "open" {
221 c.Store.SetIssueState(iss.ID, "closed")
222 }
223 c.Store.SetImportMarker(repo.ID, key, fmt.Sprintf("issue:%d", localN))
224 issues++
225 }
226 if it.Comments > 0 && localN > 0 {
227 n, err := importComments(c, g, repo, from, src, it.Number, localN, isPR)
228 if err != nil {
229 return c.fail(protocol.ExitFailure, "%v", err)
230 }
231 comments += n
232 }
233 fmt.Fprintf(c.Stderr, "%s#%d -> %s%d\n", src, it.Number, map[bool]string{true: "!", false: "#"}[isPR], localN)
234 }
235 }
236 d := map[string]any{"issues": issues, "mrs": mrs, "comments": comments, "already_imported": skipped}
237 return c.emit(d, func(w io.Writer) {
238 fmt.Fprintf(w, "imported %d issues, %d merge requests, %d comments (%d items already imported)\n",
239 issues, mrs, comments, skipped)
240 })
241}
242
243func importComments(c *Ctx, g *ghClient, repo store.Repo, from, src string, ghN, localN int64, isPR bool) (int, error) {
244 var localIssueID, localMRID int64
245 if isPR {
246 mr, err := c.Store.MRByNumber(repo.ID, localN)
247 if err != nil {
248 return 0, err
249 }
250 localMRID = mr.ID
251 } else {
252 iss, err := c.Store.IssueByNumber(repo.ID, localN)
253 if err != nil {
254 return 0, err
255 }
256 localIssueID = iss.ID
257 }
258 imported := 0
259 for page := 1; ; page++ {
260 var cs []ghComment
261 q := fmt.Sprintf("/repos/%s/issues/%d/comments?per_page=100&page=%d", url.PathEscape(from), ghN, page)
262 q = strings.ReplaceAll(q, "%2F", "/")
263 if err := g.get(q, &cs); err != nil {
264 return imported, err
265 }
266 if len(cs) == 0 {
267 return imported, nil
268 }
269 for _, cm := range cs {
270 key := fmt.Sprintf("ghc:%d", cm.ID)
271 if _, seen, err := c.Store.ImportMarker(repo.ID, key); err != nil {
272 return imported, err
273 } else if seen {
274 continue
275 }
276 body := fmt.Sprintf("> @%s, %s\n\n%s", cm.User.Login, ghDate(cm.CreatedAt), cm.Body)
277 var err error
278 if isPR {
279 err = c.Store.AddMRComment(localMRID, c.User.ID, body)
280 } else {
281 err = c.Store.AddIssueComment(localIssueID, c.User.ID, body)
282 }
283 if err != nil {
284 return imported, err
285 }
286 c.Store.SetImportMarker(repo.ID, key, "")
287 imported++
288 }
289 }
290}
internal/gitutil/messages.go +5
@@ -8,6 +8,11 @@ import (
88
99 const zeroSHA = "0000000000000000000000000000000000000000"
1010
11// HasCommit reports whether sha names a commit object present in dir.
12func HasCommit(dir, sha string) bool {
13 return exec.Command("git", "-C", dir, "cat-file", "-e", sha+"^{commit}").Run() == nil
14}
15
1116 type CommitMsg struct {
1217 SHA string
1318 Message string
internal/store/importmarkers.go added +26
@@ -0,0 +1,26 @@
1package store
2
3import (
4 "database/sql"
5 "errors"
6)
7
8// ImportMarker returns the stored value for an import progress key, and
9// whether it exists. Markers make history imports resumable: items and
10// comments already imported are skipped on re-run.
11func (s *Store) ImportMarker(repoID int64, key string) (string, bool, error) {
12 var v string
13 err := s.DB.QueryRow(
14 "SELECT value FROM import_markers WHERE repo_id = ? AND key = ?", repoID, key).Scan(&v)
15 if errors.Is(err, sql.ErrNoRows) {
16 return "", false, nil
17 }
18 return v, err == nil, err
19}
20
21func (s *Store) SetImportMarker(repoID int64, key, value string) error {
22 _, err := s.DB.Exec(
23 "INSERT INTO import_markers (repo_id, key, value) VALUES (?, ?, ?) ON CONFLICT (repo_id, key) DO UPDATE SET value = excluded.value",
24 repoID, key, value)
25 return err
26}
internal/store/migrations/0016_import_markers.down.sql added +1
@@ -0,0 +1 @@
1DROP TABLE import_markers;
internal/store/migrations/0016_import_markers.up.sql added +6
@@ -0,0 +1,6 @@
1CREATE TABLE import_markers (
2 repo_id INTEGER NOT NULL REFERENCES repos(id) ON DELETE CASCADE,
3 key TEXT NOT NULL,
4 value TEXT NOT NULL DEFAULT '',
5 PRIMARY KEY (repo_id, key)
6);