krz/gitbay

A CLI-first git forge. cli forge git self-hosted https://gitbay.org

Commit 2d315865c1

2d315865c138cf710e59c97488c38e6fa4763f56

parent: b1cab2086b

Verified · cmc

cmc <hello@cleberg.net> · 2026-09-29 05:46 UTC 

repo symbols and admin symbols reindex

repo symbols finds definitions by exact name or prefix, ranked exact,
prefix, then case-insensitive, paged with --limit/--cursor. Only the
default branch is indexed; --ref must name the indexed tree. admin
symbols reindex queues a forced rebuild.

Ref #293

Layout: unified · split

cmd/gitbay/main.go +4
@@ -146,6 +146,9 @@ func newRoot() *cobra.Command {
146146 group("mr", "merge requests in any repository (audited)",
147147 pass("prune", passOpts{server: []string{"admin", "mr", "prune"}}),
148148 ),
149 group("symbols", "symbol indexes",
150 pass("reindex", passOpts{server: []string{"admin", "symbols", "reindex"}}),
151 ),
149152 ),
150153 manCmd(root),
151154 )
@@ -553,6 +556,7 @@ func repoCmd() *cobra.Command {
553556 pass("fork", passOpts{server: []string{"repo", "fork"}, needsRepo: true}),
554557 pass("search", passOpts{server: []string{"repo", "search"}}),
555558 pass("grep", passOpts{server: []string{"repo", "grep"}, needsRepo: true}),
559 pass("symbols", passOpts{server: []string{"repo", "symbols"}, needsRepo: true}),
556560 pass("diff", passOpts{server: []string{"repo", "diff"}, needsRepo: true}),
557561 pass("tree", passOpts{server: []string{"repo", "tree"}, needsRepo: true}),
558562 pass("cat", passOpts{server: []string{"repo", "cat"}, needsRepo: true}),
cmd/gitbay/summaries_gen.go +2
@@ -17,6 +17,7 @@ var summaries = map[string]string{
1717 "admin runners remove": "drop a key's runner heartbeat row, e.g. one that polled once by mistake (instance admins)",
1818 "admin runners": "the build queue and runner accounts: last poll, scope, the build each holds (instance admins)",
1919 "admin stats": "instance statistics: counts and per-repository disk usage",
20 "admin symbols reindex": "rebuild a repository's symbol index, even when its tree is already indexed (instance admins)",
2021 "admin user create": "create an account, optionally with a key and a verified address (instance admins)",
2122 "admin user delete": "delete an account that anchors nothing (keys, emails and sessions go with it)",
2223 "admin user demote": "remove instance admin from an account (never the last one)",
@@ -207,6 +208,7 @@ var summaries = map[string]string{
207208 "repo settings visibility": "set repository visibility",
208209 "repo settings website": "set the repository website",
209210 "repo show": "show repository details",
211 "repo symbols": "find where a name is defined, from the default branch's symbol index",
210212 "repo topics add": "add topics",
211213 "repo topics remove": "remove topics",
212214 "repo topics": "list topics",
e2e/readonly_test.go +8
@@ -130,6 +130,7 @@ func TestReadOnlyCommandsWriteNothing(t *testing.T) {
130130 "repo cat": {"alice/app", "f.go"},
131131 "repo blame": {"alice/app", "f.go"},
132132 "repo grep": {"alice/app", "hello"},
133 "repo symbols": {"alice/app", "app"},
133134 "repo diff": {"alice/app", "main", "feat"},
134135 "repo commit": {"alice/app", sha},
135136 "repo download": {"alice/app"},
@@ -186,6 +187,13 @@ func TestReadOnlyCommandsWriteNothing(t *testing.T) {
186187 // an SGR sequence escaping into plain output.
187188 binaryOutput := map[string]bool{"repo download": true}
188189
190 // The push to main asked for a symbol index; the worker writing it
191 // mid-loop would be blamed on whichever read ran then.
192 waitFor(t, "symbol index", func() bool {
193 _, _, code := inst.ssh(t, aliceKey, "", "repo", "symbols", "alice/app", "app")
194 return code == 0
195 })
196
189197 dbPath := filepath.Join(inst.root, "gitbay.db")
190198 before := dbFingerprint(t, dbPath)
191199 for _, cmd := range control.Commands() {
e2e/symbols_test.go added +66
@@ -0,0 +1,66 @@
1package e2e
2
3import (
4 "encoding/json"
5 "os"
6 "path/filepath"
7 "strings"
8 "testing"
9)
10
11// A push to the default branch is indexed in the background, and
12// repo symbols then finds a Go function where it is defined (#293).
13func TestSymbolsIndexAfterPush(t *testing.T) {
14 t.Setenv("GITBAY_SYMBOLS_TICK", "200ms")
15 inst := startInstanceWith(t, "[web]\nmode = \"accounts\"\n")
16 aliceKey := inst.newKey(t, "alice")
17 bobKey := inst.newKey(t, "bob")
18 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
19 inst.admin(t, "admin", "user", "create", "bob", "--key", bobKey+".pub")
20 if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app", "--private"); code != 0 {
21 t.Fatalf("repo create: %s", errOut)
22 }
23
24 if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "symbols", "alice/app", "Serve"); code != 3 ||
25 !strings.Contains(errOut, "no symbol index") {
26 t.Fatalf("before any push: exit %d: %s", code, errOut)
27 }
28
29 work := t.TempDir()
30 env := inst.gitEnv(aliceKey)
31 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
32 dir := filepath.Join(work, "w")
33 os.WriteFile(filepath.Join(dir, "server.go"), []byte("package app\n\n// Serve runs.\nfunc Serve() {}\n"), 0o644)
34 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
35 mustGit(t, dir, env, "add", ".")
36 mustGit(t, dir, env, "commit", "-q", "-m", "base")
37 mustGit(t, dir, env, "push", "-q", "origin", "main")
38
39 var out string
40 waitFor(t, "symbol index", func() bool {
41 var code int
42 out, _, code = inst.ssh(t, aliceKey, "", "repo", "symbols", "alice/app", "Serve", "--json")
43 return code == 0
44 })
45 var res struct {
46 Data []struct {
47 Name string `json:"name"`
48 Kind string `json:"kind"`
49 Path string `json:"path"`
50 Line int `json:"line"`
51 } `json:"data"`
52 }
53 if err := json.Unmarshal([]byte(out), &res); err != nil {
54 t.Fatalf("%v: %s", err, out)
55 }
56 if len(res.Data) != 1 || res.Data[0].Name != "Serve" || res.Data[0].Kind != "function" ||
57 res.Data[0].Path != "server.go" || res.Data[0].Line != 4 {
58 t.Fatalf("repo symbols: %s", out)
59 }
60
61 // The index follows read access: to bob the repository does not exist.
62 if _, errOut, code := inst.ssh(t, bobKey, "", "repo", "symbols", "alice/app", "Serve"); code != 3 ||
63 !strings.Contains(errOut, "not found") {
64 t.Fatalf("outsider: exit %d: %s", code, errOut)
65 }
66}
internal/control/symbols.go added +175
@@ -0,0 +1,175 @@
1package control
2
3import (
4 "errors"
5 "fmt"
6 "io"
7 "strconv"
8 "strings"
9
10 "gitbay.org/gitbay/internal/gitutil"
11 "gitbay.org/gitbay/internal/policy"
12 "gitbay.org/gitbay/internal/protocol"
13 "gitbay.org/gitbay/internal/store"
14 "gitbay.org/gitbay/internal/symbols"
15)
16
17func init() {
18 register(Command{Path: []string{"repo", "symbols"},
19 Summary: "find where a name is defined, from the default branch's symbol index",
20 Usage: "repo symbols <owner/name> [--ref <ref>] [--kind <kind>] [--limit <n>] [--cursor <c>] <query>",
21 Flags: []Flag{
22 {"--ref", "<ref>", "a ref whose tree is the indexed one; only the default branch is indexed", "the default branch"},
23 {"--kind", "<kind>", "only this kind: " + strings.Join(symbols.Kinds, ", "), ""},
24 {"--limit", "<n>", "rows per page", ""},
25 {"--cursor", "<c>", "continue from the previous page", ""},
26 },
27 Examples: []string{
28 "repo symbols krz/gitbay Dispatch",
29 "repo symbols krz/gitbay --kind method Ctx.",
30 },
31 ReadOnly: true, Run: runRepoSymbols})
32 register(Command{Path: []string{"admin", "symbols", "reindex"},
33 Summary: "rebuild a repository's symbol index, even when its tree is already indexed (instance admins)",
34 Usage: "admin symbols reindex <owner/name>",
35 Examples: []string{"admin symbols reindex krz/gitbay"},
36 Run: runAdminSymbolsReindex})
37}
38
39// symbolsUnpaged caps a listing given without --limit or --cursor.
40const symbolsUnpaged = maxPageLimit
41
42type symbolOut struct {
43 Name string `json:"name"`
44 Kind string `json:"kind"`
45 Path string `json:"path"`
46 Line int `json:"line"`
47}
48
49func runRepoSymbols(c *Ctx, args []string) int {
50 args, p, code := parsePageFlags(c, args, "symbol", false)
51 if code >= 0 {
52 return code
53 }
54 f, err := c.parseArgs(args, flagSpec{Values: []string{"--ref", "--kind"}, MaxPos: 2, Usage: c.Cmd.Usage})
55 if err != nil {
56 return c.fail(protocol.ExitUsage, "%v", err)
57 }
58 path, query, ref, kind := f.pos(0), f.pos(1), f.Value("--ref"), f.Value("--kind")
59 if path == "" || query == "" {
60 return c.usage()
61 }
62 if len(query) > maxQueryLen {
63 return c.fail(protocol.ExitUsage, "query must be 1 to %d characters", maxQueryLen)
64 }
65 if kind != "" && !symbols.ValidKind(kind) {
66 return c.fail(protocol.ExitUsage, "--kind must be one of %s", strings.Join(symbols.Kinds, ", "))
67 }
68 repo, code := resolveRepo(c, path, policy.CanRead)
69 if code >= 0 {
70 return code
71 }
72 idx, err := c.Store.SymbolIndexFor(repo.ID)
73 if errors.Is(err, store.ErrNotFound) {
74 return c.fail(protocol.ExitNotFound, "%s has no symbol index yet; one is built after a push to %s", repo.Path(), repo.DefaultBranch)
75 } else if err != nil {
76 return c.fail(protocol.ExitFailure, "%v", err)
77 }
78 if ref != "" && ref != repo.DefaultBranch {
79 dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
80 if !IndexedTree(dir, ref, idx) {
81 return c.fail(protocol.ExitNotFound, "only the default branch, %s, is indexed; %s is not at the indexed tree", repo.DefaultBranch, ref)
82 }
83 }
84 if idx.State == "failed" {
85 return c.fail(protocol.ExitFailure, "the symbol index of %s failed: %s", repo.Path(), idx.Note)
86 }
87 var after int64
88 if p.key != "" {
89 cursorIdx, id, ok := parseSymbolCursor(p.key)
90 if !ok {
91 return c.fail(protocol.ExitUsage, "bad cursor")
92 }
93 if cursorIdx != idx.ID {
94 return c.fail(protocol.ExitUsage, "the symbol index was rebuilt since that cursor; start again without --cursor")
95 }
96 after = id
97 }
98 limit := p.queryLimit()
99 if !p.active {
100 limit = symbolsUnpaged + 1
101 }
102 rows, err := c.Store.SearchSymbols(idx.ID, query, kind, limit, after)
103 if err != nil {
104 return c.fail(protocol.ExitFailure, "%v", err)
105 }
106 capped := !p.active && len(rows) > symbolsUnpaged
107 if capped {
108 rows = rows[:symbolsUnpaged]
109 }
110 rows, next := trimPage(p, rows, "symbol", func(r store.SymbolRow) string {
111 return strconv.FormatInt(idx.ID, 10) + "." + strconv.FormatInt(r.ID, 10)
112 })
113 var ds []symbolOut
114 for _, r := range rows {
115 ds = append(ds, symbolOut{r.Name, r.Kind, r.Path, r.Line})
116 }
117 return c.emitPage(p, ds, next, func(w io.Writer) {
118 tb := c.table(w, "NAME", "KIND", "LOCATION")
119 for _, d := range ds {
120 tb.row(cRef(d.Name), cText(d.Kind), cFlex(fmt.Sprintf("%s:%d", d.Path, d.Line)))
121 }
122 tb.flush()
123 if capped {
124 fmt.Fprintf(c.Stderr, "first %d matches; page with --limit and --cursor\n", symbolsUnpaged)
125 }
126 if idx.State == "partial" {
127 fmt.Fprintf(c.Stderr, "the index is partial: %s\n", idx.Note)
128 }
129 })
130}
131
132// parseSymbolCursor reads "<index id>.<row id>". The index id makes a
133// cursor from before a rebuild fail rather than page through the new
134// index from an unrelated row.
135func parseSymbolCursor(key string) (int64, int64, bool) {
136 a, b, ok := strings.Cut(key, ".")
137 if !ok {
138 return 0, 0, false
139 }
140 idx, err1 := strconv.ParseInt(a, 10, 64)
141 id, err2 := strconv.ParseInt(b, 10, 64)
142 return idx, id, err1 == nil && err2 == nil && id > 0
143}
144
145// IndexedTree reports whether ref names a commit whose tree is the one
146// idx was built from: the default branch's head when the index is
147// current, or any other commit with the same content. The blob view uses
148// it to decide whether its names can link into the index.
149func IndexedTree(dir, ref string, idx store.SymbolIndex) bool {
150 sha, err := gitutil.ResolveRef(dir, ref)
151 if err != nil {
152 return false
153 }
154 if sha == idx.Commit {
155 return true
156 }
157 tree, err := gitutil.ResolveTree(dir, sha)
158 return err == nil && tree == idx.Tree
159}
160
161func runAdminSymbolsReindex(c *Ctx, args []string) int {
162 if len(args) != 1 {
163 return c.usage()
164 }
165 repo, code := adminRepo(c, args[0])
166 if code >= 0 {
167 return code
168 }
169 if err := c.Store.RequestSymbolIndex(repo.ID, true); err != nil {
170 return c.fail(protocol.ExitFailure, "%v", err)
171 }
172 return c.emit(map[string]string{"repo": repo.Path(), "state": "queued"}, func(w io.Writer) {
173 fmt.Fprintf(w, "queued a rebuild of the symbol index of %s\n", repo.Path())
174 })
175}
internal/control/symbols_test.go added +143
@@ -0,0 +1,143 @@
1package control
2
3import (
4 "bytes"
5 "encoding/json"
6 "strings"
7 "testing"
8
9 "gitbay.org/gitbay/internal/protocol"
10 "gitbay.org/gitbay/internal/store"
11)
12
13func symbolsFixture(t *testing.T) (*store.Store, store.Repo, store.User, store.User) {
14 t.Helper()
15 st, err := store.Open(":memory:")
16 if err != nil {
17 t.Fatal(err)
18 }
19 t.Cleanup(func() { st.Close() })
20 if err := st.MigrateUp(); err != nil {
21 t.Fatal(err)
22 }
23 aliceID, _ := st.CreateUser("alice", false)
24 bobID, _ := st.CreateUser("bob", false)
25 id, err := st.CreateRepo("user", aliceID, "secret", "private")
26 if err != nil {
27 t.Fatal(err)
28 }
29 repo, _ := st.RepoByID(id)
30 var rows []store.SymbolRow
31 for _, n := range []string{"Parse", "ParseAll", "ParseArgs", "Parser", "parse"} {
32 rows = append(rows, store.SymbolRow{Name: n, Key: n, Kind: "function", Path: "p.go", Line: len(rows) + 1})
33 }
34 rows = append(rows, store.SymbolRow{Name: "Parser.Run", Key: "Run", Kind: "method", Path: "p.go", Line: 40})
35 if _, err := st.ReplaceSymbolIndex(store.SymbolIndex{RepoID: id, Commit: "c0", Tree: "t0", State: "ok"}, rows); err != nil {
36 t.Fatal(err)
37 }
38 return st, repo,
39 store.User{ID: aliceID, Username: "alice"},
40 store.User{ID: bobID, Username: "bob"}
41}
42
43type symbolPage struct {
44 Data struct {
45 Items []symbolOut `json:"items"`
46 Next string `json:"next"`
47 } `json:"data"`
48}
49
50func runSymbols(t *testing.T, st *store.Store, u store.User, argv ...string) (int, string, string) {
51 t.Helper()
52 var out, errOut bytes.Buffer
53 c := &Ctx{User: u, Scope: "full", Store: st, Stdout: &out, Stderr: &errOut, JSON: true}
54 c.Cfg.Server.Root = t.TempDir()
55 code := Dispatch(c, argv)
56 if code != protocol.ExitOK {
57 // In JSON mode a refusal is the output's error field.
58 return code, out.String(), out.String() + errOut.String()
59 }
60 return code, out.String(), errOut.String()
61}
62
63func TestRepoSymbolsPages(t *testing.T) {
64 st, repo, alice, _ := symbolsFixture(t)
65 var names []string
66 cursor := ""
67 for pages := 0; ; pages++ {
68 argv := []string{"repo", "symbols", repo.Path(), "Parse", "--limit", "2"}
69 if cursor != "" {
70 argv = append(argv, "--cursor", cursor)
71 }
72 code, out, errOut := runSymbols(t, st, alice, argv...)
73 if code != protocol.ExitOK {
74 t.Fatalf("exit %d: %s", code, errOut)
75 }
76 var p symbolPage
77 if err := json.Unmarshal([]byte(out), &p); err != nil {
78 t.Fatalf("%v: %s", err, out)
79 }
80 for _, it := range p.Data.Items {
81 names = append(names, it.Name)
82 }
83 if p.Data.Next == "" {
84 break
85 }
86 if pages > 5 {
87 t.Fatal("paging does not end")
88 }
89 cursor = p.Data.Next
90 }
91 if got, want := strings.Join(names, " "), "Parse ParseAll ParseArgs Parser Parser.Run parse"; got != want {
92 t.Fatalf("got %q, want %q", got, want)
93 }
94
95 // A cursor from before a rebuild is refused rather than followed.
96 st.ReplaceSymbolIndex(store.SymbolIndex{RepoID: repo.ID, Commit: "c1", Tree: "t1", State: "ok"},
97 []store.SymbolRow{{Name: "Parse", Key: "Parse", Kind: "function", Path: "p.go", Line: 1}})
98 code, _, errOut := runSymbols(t, st, alice, "repo", "symbols", repo.Path(), "Parse", "--cursor", cursor)
99 if code != protocol.ExitUsage || !strings.Contains(errOut, "rebuilt") {
100 t.Fatalf("stale cursor: exit %d: %s", code, errOut)
101 }
102}
103
104func TestRepoSymbolsRefusals(t *testing.T) {
105 st, repo, alice, bob := symbolsFixture(t)
106 for _, tc := range []struct {
107 name string
108 user store.User
109 argv []string
110 code int
111 msg string
112 }{
113 {"outsider sees no repository", bob, []string{"repo", "symbols", repo.Path(), "Parse"}, protocol.ExitNotFound, "not found"},
114 {"another ref", alice, []string{"repo", "symbols", repo.Path(), "--ref", "feature", "Parse"}, protocol.ExitNotFound, "only the default branch"},
115 {"unknown kind", alice, []string{"repo", "symbols", repo.Path(), "--kind", "widget", "Parse"}, protocol.ExitUsage, "--kind"},
116 {"no query", alice, []string{"repo", "symbols", repo.Path()}, protocol.ExitUsage, ""},
117 {"reindex needs an admin", alice, []string{"admin", "symbols", "reindex", repo.Path()}, protocol.ExitDenied, "admin"},
118 } {
119 t.Run(tc.name, func(t *testing.T) {
120 code, _, errOut := runSymbols(t, st, tc.user, tc.argv...)
121 if code != tc.code || !strings.Contains(errOut, tc.msg) {
122 t.Fatalf("exit %d, want %d with %q: %s", code, tc.code, tc.msg, errOut)
123 }
124 })
125 }
126
127 code, _, errOut := runSymbols(t, st, alice, "repo", "symbols", repo.Path(), "--ref", repo.DefaultBranch, "--kind", "method", "Run")
128 if code != protocol.ExitOK {
129 t.Fatalf("default branch by name: exit %d: %s", code, errOut)
130 }
131}
132
133func TestAdminSymbolsReindexQueuesAForcedBuild(t *testing.T) {
134 st, repo, _, _ := symbolsFixture(t)
135 code, _, errOut := runSymbols(t, st, rootUser(t, st), "admin", "symbols", "reindex", repo.Path())
136 if code != protocol.ExitOK {
137 t.Fatalf("exit %d: %s", code, errOut)
138 }
139 reqs, err := st.SymbolRequests()
140 if err != nil || len(reqs) != 1 || reqs[0].RepoID != repo.ID || !reqs[0].Force {
141 t.Fatalf("requests = %+v, %v", reqs, err)
142 }
143}