Commit `f6cb7d04f1`

f6cb7d04f133ff5d911efe3f741e3b95330ffc1d

parent: d6d78eea2f

Verified · cmc

cmc <hello@cleberg.net> · 2026-08-23 23:36 UTC

repo import: server-side mirror from a foreign URL

- repo import <owner/name> --from <url> [--private] [--token-stdin]:
  fetches heads, tags, and notes into a fresh bare repo (hooks wired
  via core.hooksPath), detects the remote default branch via ls-remote
  --symref and sets HEAD accordingly; clone_timeout applies
- credentials only via --token-stdin: the token reaches git through a
  GIT_ASKPASS helper reading the environment — never argv, never the
  URL, never the database; URLs with embedded credentials are refused
- scheme allowlist https/http/git (file:// and ssh:// refused); import
  lands only under the caller's own account; git data only, stated in
  the output; failed imports clean up both the row and the directory
- CLI passthrough with stdin wired for --token-stdin
- e2e: import over http and git:// from the instance's own transports,
  branch/tag/HEAD fidelity, hooks functional on the imported repo
  (protected-branch force-push refused), all refusal cases, failure
  cleanup, and token non-leakage into repo config; verified against a
  real GitHub repository manually

Layout: unified · split

cmd/forge/main.go +36

		@@ -194,6 +194,7 @@ func repoCmd() *cobra.Command {
194	194	pass("delete", "delete a repository (--yes)", passOpts{server: []string{"repo", "delete"}, needsRepo: true}),
195	195	pass("fork", "fork a repository under your account", passOpts{server: []string{"repo", "fork"}, needsRepo: true}),
196	196	local("clone", "clone via ssh: forge repo clone <owner/name> [dir]", cmdRepoClone),
	197	importCmd(),
197	198	group("access", "manage access grants",
198	199	pass("grant", "grant access: ... <user> read\|write\|admin", passOpts{server: []string{"repo", "access", "grant"}, needsRepo: true}),
199	200	pass("revoke", "revoke access: ... <user>", passOpts{server: []string{"repo", "access", "revoke"}, needsRepo: true}),
		@@ -239,6 +240,41 @@ func mrCmd() *cobra.Command {
239	240	)
240	241	}
241	242
	243	// importCmd passes repo import through with stdin wired for --token-stdin.
	244	func importCmd() *cobra.Command {
	245	return &cobra.Command{
	246	Use: "import",
	247	Short: "server-side mirror of a foreign repo: forge repo import <owner/name> --from <url> [--private] [--token-stdin]",
	248	DisableFlagParsing: true,
	249	RunE: func(cmd *cobra.Command, args []string) error {
	250	for _, a := range args {
	251	if a == "--help" \|\| a == "-h" {
	252	return cmd.Help()
	253	}
	254	}
	255	t, err := resolveTarget()
	256	if err != nil {
	257	return err
	258	}
	259	var stdin io.Reader = strings.NewReader("")
	260	if usesTokenStdin(args) {
	261	stdin = os.Stdin
	262	}
	263	os.Exit(runSSH(t, append([]string{"repo", "import"}, args...), stdin))
	264	return nil
	265	},
	266	}
	267	}
	268
	269	func usesTokenStdin(args []string) bool {
	270	for _, a := range args {
	271	if a == "--token-stdin" {
	272	return true
	273	}
	274	}
	275	return false
	276	}
	277
242	278	func webCmd() *cobra.Command {
243	279	return group("web", "browser session",
244	280	pass("login", "mint a one-time browser login URL over ssh", passOpts{server: []string{"web", "login"}}),

e2e/import_test.go added +135

		@@ -0,0 +1,135 @@
	1	package e2e
	2
	3	import (
	4	"fmt"
	5	"os"
	6	"path/filepath"
	7	"strings"
	8	"testing"
	9	)
	10
	11	func TestRepoImport(t *testing.T) {
	12	inst := startInstance(t)
	13	aliceKey := inst.newKey(t, "alice")
	14	inst.admin(t, "admin", "user", "create", "alice",
	15	"--key", aliceKey+".pub", "--email", "alice@example.test", "--verified")
	16
	17	// Source repo with a non-"main" default branch, a tag, and two commits.
	18	if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/src"); code != 0 {
	19	t.Fatalf("repo create: %s", errOut)
	20	}
	21	work := t.TempDir()
	22	env := inst.gitEnv(aliceKey)
	23	mustGit(t, work, env, "clone", inst.sshURL("alice/src"), "w")
	24	dir := filepath.Join(work, "w")
	25	os.WriteFile(filepath.Join(dir, "code.txt"), []byte("v1\n"), 0o644)
	26	mustGit(t, dir, env, "checkout", "-q", "-b", "trunk")
	27	mustGit(t, dir, env, "add", ".")
	28	mustGit(t, dir, env, "commit", "-q", "-m", "first")
	29	mustGit(t, dir, env, "tag", "v1.0")
	30	mustGit(t, dir, env, "commit", "-q", "--allow-empty", "-m", "second")
	31	mustGit(t, dir, env, "push", "-q", "origin", "trunk", "v1.0")
	32
	33	// Point the source repo's HEAD at trunk so the remote advertises it.
	34	srcBare := filepath.Join(inst.root, "repos", "alice", "src.git")
	35	mustGit(t, srcBare, env, "symbolic-ref", "HEAD", "refs/heads/trunk")
	36
	37	// Import over HTTP from our own instance (public smart HTTP).
	38	httpURL := fmt.Sprintf("http://127.0.0.1:%d/alice/src.git", inst.httpPort)
	39	out, errOut, code := inst.ssh(t, aliceKey, "", "repo", "import", "alice/mirror", "--from", httpURL, "--private")
	40	if code != 0 {
	41	t.Fatalf("import: exit %d\n%s%s", code, out, errOut)
	42	}
	43	if !strings.Contains(out, "imported alice/mirror (private, default trunk)") {
	44	t.Fatalf("import output: %s", out)
	45	}
	46	if !strings.Contains(out, "git data only") {
	47	t.Fatalf("import note missing: %s", out)
	48	}
	49
	50	// Everything came across: both commits, the tag, and the default branch.
	51	logOut, _, code := inst.ssh(t, aliceKey, "", "repo", "log", "alice/mirror")
	52	if code != 0 \|\| !strings.Contains(logOut, "second") \|\| !strings.Contains(logOut, "first") {
	53	t.Fatalf("imported log: %d\n%s", code, logOut)
	54	}
	55	mirrorBare := filepath.Join(inst.root, "repos", "alice", "mirror.git")
	56	tags := mustGit(t, mirrorBare, env, "tag", "--list")
	57	if !strings.Contains(tags, "v1.0") {
	58	t.Fatalf("tag not imported: %q", tags)
	59	}
	60	head := strings.TrimSpace(mustGit(t, mirrorBare, env, "symbolic-ref", "HEAD"))
	61	if head != "refs/heads/trunk" {
	62	t.Fatalf("imported HEAD = %s", head)
	63	}
	64	showOut, _, _ := inst.ssh(t, aliceKey, "", "repo", "show", "alice/mirror")
	65	if !strings.Contains(showOut, "private") \|\| !strings.Contains(showOut, "default: trunk") {
	66	t.Fatalf("repo show after import: %s", showOut)
	67	}
	68
	69	// The imported repo has working hooks (core.hooksPath was set): a
	70	// protected-branch force-push is refused.
	71	if _, errOut, code = inst.ssh(t, aliceKey, "", "repo", "settings", "protect", "alice/mirror", "trunk"); code != 0 {
	72	t.Fatalf("protect: %s", errOut)
	73	}
	74	mWork := t.TempDir()
	75	mustGit(t, mWork, env, "clone", inst.sshURL("alice/mirror"), "m")
	76	mDir := filepath.Join(mWork, "m")
	77	mustGit(t, mDir, env, "commit", "-q", "--amend", "--allow-empty", "-m", "rewrite")
	78	pushOut, pushCode := gitRun(t, mDir, env, "push", "--force", "origin", "trunk")
	79	if pushCode == 0 \|\| !strings.Contains(pushOut, "force-push refused") {
	80	t.Fatalf("hooks not wired on imported repo:\n%s", pushOut)
	81	}
	82
	83	// Import over git:// too.
	84	if _, errOut, code = inst.ssh(t, aliceKey, "", "repo", "settings", "git-daemon", "alice/src", "on"); code != 0 {
	85	t.Fatalf("git-daemon on: %s", errOut)
	86	}
	87	gitURL := fmt.Sprintf("git://127.0.0.1:%d/alice/src.git", inst.gitPort)
	88	if _, errOut, code = inst.ssh(t, aliceKey, "", "repo", "import", "alice/mirror2", "--from", gitURL); code != 0 {
	89	t.Fatalf("git:// import: %s", errOut)
	90	}
	91
	92	// Refusals: bad scheme, credentials in URL, existing name, foreign owner.
	93	cases := []struct {
	94	args []string
	95	want string
	96	}{
	97	{[]string{"repo", "import", "alice/x", "--from", "file:///etc"}, "https://, http://, and git://"},
	98	{[]string{"repo", "import", "alice/x", "--from", "https://token@github.com/a/b"}, "--token-stdin"},
	99	{[]string{"repo", "import", "alice/mirror", "--from", httpURL}, "already exists"},
	100	{[]string{"repo", "import", "bob/x", "--from", httpURL}, "your own account"},
	101	}
	102	for _, tc := range cases {
	103	_, errOut, code := inst.ssh(t, aliceKey, "", tc.args...)
	104	if code == 0 \|\| !strings.Contains(errOut, tc.want) {
	105	t.Errorf("%v: exit %d, stderr %q (want %q)", tc.args, code, errOut, tc.want)
	106	}
	107	}
	108
	109	// A failed import leaves nothing behind.
	110	_, _, code = inst.ssh(t, aliceKey, "", "repo", "import", "alice/gone", "--from",
	111	fmt.Sprintf("http://127.0.0.1:%d/alice/nonexistent.git", inst.httpPort))
	112	if code == 0 {
	113	t.Fatal("import of nonexistent source succeeded")
	114	}
	115	if _, _, code = inst.ssh(t, aliceKey, "", "repo", "show", "alice/gone"); code != 3 {
	116	t.Fatalf("failed import left a repo behind: exit %d, want 3", code)
	117	}
	118	if _, err := os.Stat(filepath.Join(inst.root, "repos", "alice", "gone.git")); !os.IsNotExist(err) {
	119	t.Fatal("failed import left a directory behind")
	120	}
	121
	122	// --token-stdin consumes a token from stdin without leaking it: the
	123	// fetch works (token unused by our anonymous endpoint, but the askpass
	124	// plumbing must not break it) and the token string appears nowhere in
	125	// the repo config.
	126	_, errOut, code = inst.ssh(t, aliceKey, "s3cr3t-token\n", "repo", "import", "alice/mirror3",
	127	"--from", httpURL, "--token-stdin")
	128	if code != 0 {
	129	t.Fatalf("token-stdin import: %s", errOut)
	130	}
	131	cfgRaw, _ := os.ReadFile(filepath.Join(inst.root, "repos", "alice", "mirror3.git", "config"))
	132	if strings.Contains(string(cfgRaw), "s3cr3t") {
	133	t.Fatal("token leaked into repo config")
	134	}
	135	}

internal/control/import.go added +154

		@@ -0,0 +1,154 @@
	1	package control
	2
	3	import (
	4	"bufio"
	5	"context"
	6	"fmt"
	7	"io"
	8	"os"
	9	"path/filepath"
	10	"strings"
	11	"time"
	12
	13	"github.com/krazywarez/forge/internal/gitutil"
	14	"github.com/krazywarez/forge/internal/policy"
	15	"github.com/krazywarez/forge/internal/protocol"
	16	)
	17
	18	func init() {
	19	register(Command{Path: []string{"repo", "import"},
	20	Summary: "server-side mirror of a foreign repository: repo import <owner/name> --from <url> [--private] [--token-stdin]",
	21	ReadsStdin: true, Run: runRepoImport})
	22	}
	23
	24	// askpassScript answers git's credential prompts from the environment, so
	25	// the token never appears on a command line or in a URL. Username prompts
	26	// get a placeholder (GitHub and GitLab ignore it for token auth).
	27	const askpassScript = `#!/bin/sh
	28	case "$1" in
	29	Username*) echo "x-access-token" ;;
	30	*) echo "${FORGE_IMPORT_TOKEN}" ;;
	31	esac
	32	`
	33
	34	func runRepoImport(c *Ctx, args []string) int {
	35	var path, from string
	36	private := false
	37	tokenStdin := false
	38	for i := 0; i < len(args); i++ {
	39	switch args[i] {
	40	case "--from":
	41	if i+1 >= len(args) {
	42	return c.fail(protocol.ExitUsage, "--from requires a URL")
	43	}
	44	from = args[i+1]
	45	i++
	46	case "--private":
	47	private = true
	48	case "--token-stdin":
	49	tokenStdin = true
	50	default:
	51	if path != "" {
	52	return c.fail(protocol.ExitUsage, "unexpected argument %q", args[i])
	53	}
	54	path = args[i]
	55	}
	56	}
	57	if path == "" \|\| from == "" {
	58	return c.fail(protocol.ExitUsage, "usage: repo import <owner/name> --from <url> [--private] [--token-stdin]")
	59	}
	60	owner, name, ok := strings.Cut(path, "/")
	61	if !ok \|\| owner != c.User.Username {
	62	return c.fail(protocol.ExitDenied, "imports land under your own account: %s/<name>", c.User.Username)
	63	}
	64	if err := policy.ValidateName(name); err != nil {
	65	return c.fail(protocol.ExitUsage, "%v", err)
	66	}
	67
	68	// Scheme allowlist. file:// (and anything else local) would read the
	69	// server's filesystem; ssh:// would use the server's own keys.
	70	switch {
	71	case strings.HasPrefix(from, "https://"), strings.HasPrefix(from, "http://"), strings.HasPrefix(from, "git://"):
	72	default:
	73	return c.fail(protocol.ExitUsage, "import supports https://, http://, and git:// URLs only")
	74	}
	75	if strings.ContainsAny(from, "@") {
	76	// Credentials belong on stdin, not in the URL where they would
	77	// land in process listings and logs.
	78	return c.fail(protocol.ExitUsage, "do not embed credentials in the URL; use --token-stdin")
	79	}
	80
	81	// The token is read from stdin and handed to git via GIT_ASKPASS and
	82	// the environment — never argv, never the database, never a log line.
	83	var env []string
	84	if tokenStdin {
	85	token, err := bufio.NewReader(io.LimitReader(c.Stdin, 4096)).ReadString('\n')
	86	if err != nil && err != io.EOF {
	87	return c.fail(protocol.ExitFailure, "reading token: %v", err)
	88	}
	89	token = strings.TrimSpace(token)
	90	if token == "" {
	91	return c.fail(protocol.ExitUsage, "--token-stdin given but stdin held no token")
	92	}
	93	askpass := filepath.Join(c.Cfg.Server.Root, "askpass.sh")
	94	if err := os.WriteFile(askpass, []byte(askpassScript), 0o700); err != nil {
	95	return c.fail(protocol.ExitFailure, "%v", err)
	96	}
	97	env = []string{
	98	"GIT_ASKPASS=" + askpass,
	99	"FORGE_IMPORT_TOKEN=" + token,
	100	"GIT_TERMINAL_PROMPT=0",
	101	}
	102	} else {
	103	env = []string{"GIT_TERMINAL_PROMPT=0"}
	104	}
	105
	106	visibility := "public"
	107	if private {
	108	visibility = "private"
	109	}
	110	id, err := c.Store.CreateRepo("user", c.User.ID, name, visibility)
	111	if err != nil {
	112	return c.fail(protocol.ExitFailure, "%v", err)
	113	}
	114	dir := RepoDir(c.Cfg.Server.Root, owner, name)
	115	cleanup := func() {
	116	c.Store.DeleteRepo(id)
	117	os.RemoveAll(dir)
	118	}
	119	if err := gitutil.InitBare(dir, "main", HooksDir(c.Cfg.Server.Root)); err != nil {
	120	cleanup()
	121	return c.fail(protocol.ExitFailure, "%v", err)
	122	}
	123
	124	timeout := time.Duration(c.Cfg.Limits.CloneTimeoutSec) * time.Second
	125	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	126	defer cancel()
	127
	128	fmt.Fprintf(c.Stderr, "importing %s into %s ...\n", from, path)
	129	if err := gitutil.FetchMirror(ctx, dir, from, c.Stderr, env); err != nil {
	130	cleanup()
	131	return c.fail(protocol.ExitFailure, "import failed: %v", err)
	132	}
	133
	134	branch, err := gitutil.RemoteDefaultBranch(ctx, from, env)
	135	if err != nil {
	136	branch = "main" // remote gone quiet after the fetch; keep the default
	137	}
	138	if _, rerr := gitutil.ResolveRef(dir, "refs/heads/"+branch); rerr == nil {
	139	gitutil.SetHead(dir, branch)
	140	c.Store.UpdateDefaultBranch(id, branch)
	141	}
	142
	143	c.Store.RecordEvent(id, c.User.ID, "repo.imported", fmt.Sprintf(`{"from":%q}`, from))
	144	type out struct {
	145	Path string `json:"path"`
	146	Visibility string `json:"visibility"`
	147	DefaultBranch string `json:"default_branch"`
	148	}
	149	d := out{path, visibility, branch}
	150	return c.emit(d, func(w io.Writer) {
	151	fmt.Fprintf(w, "imported %s (%s, default %s)\nnote: git data only — issues and pull requests do not transfer\n",
	152	d.Path, d.Visibility, d.DefaultBranch)
	153	})
	154	}

internal/gitutil/gitutil.go +46

		@@ -3,6 +3,7 @@
3	3	package gitutil
4	4
5	5	import (
	6	"context"
6	7	"fmt"
7	8	"io"
8	9	"os"
		@@ -100,3 +101,48 @@ func ReadCommit(dir, sha string) ([]byte, error) {
100	101	}
101	102	return out, nil
102	103	}
	104
	105	// FetchMirror pulls all branches, tags, and notes from a foreign URL into
	106	// the bare repository at dir, forcing updates. Progress streams to errW so
	107	// an interactive caller can watch. extraEnv carries credentials via
	108	// GIT_ASKPASS; the URL itself must never contain them.
	109	func FetchMirror(ctx context.Context, dir, url string, errW io.Writer, extraEnv []string) error {
	110	cmd := exec.CommandContext(ctx, "git", "-C", dir, "fetch", "--progress", "--no-write-fetch-head", url,
	111	"+refs/heads/:refs/heads/",
	112	"+refs/tags/:refs/tags/",
	113	"+refs/notes/:refs/notes/")
	114	cmd.Env = append(os.Environ(), extraEnv...)
	115	cmd.Stderr = errW
	116	if err := cmd.Run(); err != nil {
	117	return fmt.Errorf("fetch from %s: %w", url, err)
	118	}
	119	return nil
	120	}
	121
	122	// RemoteDefaultBranch asks the remote which branch HEAD points at.
	123	func RemoteDefaultBranch(ctx context.Context, url string, extraEnv []string) (string, error) {
	124	cmd := exec.CommandContext(ctx, "git", "ls-remote", "--symref", url, "HEAD")
	125	cmd.Env = append(os.Environ(), extraEnv...)
	126	out, err := cmd.Output()
	127	if err != nil {
	128	return "", fmt.Errorf("ls-remote %s: %w", url, err)
	129	}
	130	// "ref: refs/heads/<branch>\tHEAD"
	131	for _, line := range strings.Split(string(out), "\n") {
	132	if rest, ok := strings.CutPrefix(line, "ref: refs/heads/"); ok {
	133	if branch, _, ok := strings.Cut(rest, "\t"); ok {
	134	return branch, nil
	135	}
	136	}
	137	}
	138	return "", fmt.Errorf("remote %s did not advertise a default branch", url)
	139	}
	140
	141	// SetHead points the bare repo's HEAD at a branch.
	142	func SetHead(dir, branch string) error {
	143	cmd := exec.Command("git", "-C", dir, "symbolic-ref", "HEAD", "refs/heads/"+branch)
	144	if out, err := cmd.CombinedOutput(); err != nil {
	145	return fmt.Errorf("symbolic-ref: %v\n%s", err, out)
	146	}
	147	return nil
	148	}

internal/store/repos.go +5

		@@ -219,3 +219,8 @@ func (s *Store) ListPublicRepos() ([]Repo, error) {
219	219	}
220	220	return out, rows.Err()
221	221	}
	222
	223	func (s *Store) UpdateDefaultBranch(repoID int64, branch string) error {
	224	_, err := s.DB.Exec("UPDATE repos SET default_branch = ? WHERE id = ?", branch, repoID)
	225	return err
	226	}

Commit f6cb7d04f1

Commit `f6cb7d04f1`