Commit `b1cab2086b`

b1cab2086b46d9d57b18b492ecf16061b03ff0f0

parent: ebab9139d1

Verified · cmc

cmc <hello@cleberg.net> · 2026-09-29 05:45 UTC

symbols: index definitions per default-branch tree

A background worker builds one symbol index per repository from the
default branch head, keyed by its tree id (migration). Push,
merge, default-branch change, import and pull-mirror sync queue a
request. Go is parsed with go/parser; other languages match by line.
Bounded by file size, vendored and generated paths, 200k symbols and
two minutes; failed and partial runs are recorded, not retried.

Ref #293

Layout: unified · split

cmd/gitbayd/main.go +4

		@@ -37,6 +37,7 @@ import (
37	37	"gitbay.org/gitbay/internal/seal"
38	38	"gitbay.org/gitbay/internal/sshd"
39	39	"gitbay.org/gitbay/internal/store"
	40	"gitbay.org/gitbay/internal/symbols"
40	41	"gitbay.org/gitbay/internal/toolpath"
41	42	"gitbay.org/gitbay/internal/webhook"
42	43	)
		@@ -228,6 +229,9 @@ func serveCmd() *cobra.Command {
228	229	go deps.New(st, cfg, func(owner, name string) string {
229	230	return control.RepoDir(cfg.Server.Root, owner, name)
230	231	}, buildinfo.String()).Run(whCtx)
	232	go symbols.New(st, func(owner, name string) string {
	233	return control.RepoDir(cfg.Server.Root, owner, name)
	234	}).Run(whCtx)
231	235
232	236	// One pack-generation budget for SSH, smart HTTP and git://.
233	237	// Anonymous clients ("ip:" principals) share all but one

internal/control/import.go +1

		@@ -173,6 +173,7 @@ func runRepoImport(c *Ctx, args []string) int {
173	173	gitutil.SetHead(dir, branch)
174	174	c.Store.UpdateDefaultBranch(id, branch)
175	175	}
	176	c.Store.RequestSymbolIndex(id, false)
176	177
177	178	c.Store.RecordEvent(id, c.User.ID, "repo.imported", fmt.Sprintf(`{"from":%q}`, from))
178	179	type out struct {

internal/control/mr.go +1

		@@ -1611,6 +1611,7 @@ func mergeMR(c *Ctx, repo store.Repo, mr store.MR, strategy string) int {
1611	1611	ProcessCommitMessages(c.Store, dir, repo, c.User.ID, c.Scope, targetSHA, newSHA)
1612	1612	ProcessMRDescription(c.Store, repo, mr, c.User.ID, c.Scope)
1613	1613	RecordLandedCommits(c.Store, dir, repo, targetSHA, newSHA)
	1614	c.Store.RequestSymbolIndex(repo.ID, false)
1614	1615	}
1615	1616	// A merge moves the ref directly, so it never reaches post-receive and
1616	1617	// none of the ref-update work fires on its own. The event webhooks

internal/control/refsupdated.go +2

		@@ -51,6 +51,8 @@ func RefsUpdated(st *store.Store, cfg config.Config, repoID, userID int64, scope
51	51	dir := RepoDir(cfg.Server.Root, pushedRepo.OwnerName, pushedRepo.Name)
52	52	ProcessCommitMessages(st, dir, pushedRepo, userID, scope, u.Old, u.New)
53	53	RecordLandedCommits(st, dir, pushedRepo, u.Old, u.New)
	54	// The symbol index is rebuilt by its worker, never here.
	55	st.RequestSymbolIndex(pushedRepo.ID, false)
54	56	}
55	57	// A branch push with a .gitbay/ci.yml queues one build per job.
56	58	if pushedRepoErr == nil && !u.IsDelete {

internal/control/repo.go +1

		@@ -790,6 +790,7 @@ func runSetDefaultBranch(c *Ctx, args []string) int {
790	790	if err := c.Store.UpdateDefaultBranch(repo.ID, branch); err != nil {
791	791	return c.fail(protocol.ExitFailure, "%v", err)
792	792	}
	793	c.Store.RequestSymbolIndex(repo.ID, false)
793	794	return c.emit(map[string]string{"default_branch": branch}, func(w io.Writer) {
794	795	fmt.Fprintf(w, "default branch of %s is now %s\n", repo.Path(), branch)
795	796	})

internal/gitutil/treeblobs.go added +117

		@@ -0,0 +1,117 @@
	1	package gitutil
	2
	3	import (
	4	"bufio"
	5	"bytes"
	6	"context"
	7	"errors"
	8	"fmt"
	9	"io"
	10	"os/exec"
	11	"strconv"
	12	"strings"
	13
	14	"gitbay.org/gitbay/internal/toolpath"
	15	)
	16
	17	// ListBlobs lists every blob in a tree, recursively, with its size. Name
	18	// is the path from the tree's root. Paths are read NUL-separated, so no
	19	// file name is quoted or split.
	20	func ListBlobs(ctx context.Context, dir, tree string) ([]TreeEntry, error) {
	21	out, err := exec.CommandContext(ctx, toolpath.Look("git"), "-C", dir,
	22	"ls-tree", "-r", "-l", "-z", "--end-of-options", tree).Output()
	23	if err != nil {
	24	return nil, fmt.Errorf("ls-tree %s: %w", tree, err)
	25	}
	26	var entries []TreeEntry
	27	for _, rec := range bytes.Split(out, []byte{0}) {
	28	meta, name, ok := strings.Cut(string(rec), "\t")
	29	if !ok {
	30	continue
	31	}
	32	f := strings.Fields(meta)
	33	if len(f) != 4 \|\| f[1] != "blob" {
	34	continue
	35	}
	36	// ls-tree prints a size of BAD for a blob it cannot read.
	37	size, err := strconv.ParseInt(f[3], 10, 64)
	38	if err != nil {
	39	return nil, fmt.Errorf("ls-tree %s: cannot read %s", tree, name)
	40	}
	41	entries = append(entries, TreeEntry{Mode: f[0], Type: f[1], SHA: f[2], Size: size, Name: name})
	42	}
	43	return entries, nil
	44	}
	45
	46	// CatBlobs reads blobs by id through one `git cat-file --batch`, calling
	47	// fn with each one's index in shas and its contents, in order. fn
	48	// returning false stops the read. Cancelling parent stops it too, and is
	49	// reported as parent's error.
	50	func CatBlobs(parent context.Context, dir string, shas []string, fn func(i int, data []byte) bool) error {
	51	ctx, cancel := context.WithCancel(parent)
	52	defer cancel()
	53	cmd := exec.CommandContext(ctx, toolpath.Look("git"), "-C", dir, "cat-file", "--batch")
	54	stdin, err := cmd.StdinPipe()
	55	if err != nil {
	56	return err
	57	}
	58	stdout, err := cmd.StdoutPipe()
	59	if err != nil {
	60	return err
	61	}
	62	if err := cmd.Start(); err != nil {
	63	return err
	64	}
	65	go func() {
	66	w := bufio.NewWriter(stdin)
	67	for _, s := range shas {
	68	if _, err := fmt.Fprintln(w, s); err != nil {
	69	break
	70	}
	71	}
	72	w.Flush()
	73	stdin.Close()
	74	}()
	75	r := bufio.NewReader(stdout)
	76	readErr := func() error {
	77	for i := range shas {
	78	header, err := r.ReadString('\n')
	79	if err != nil {
	80	return err
	81	}
	82	// <sha> <type> <size>, or <sha> missing
	83	f := strings.Fields(header)
	84	if len(f) != 3 {
	85	return fmt.Errorf("cat-file: %s", strings.TrimSpace(header))
	86	}
	87	size, err := strconv.ParseInt(f[2], 10, 64)
	88	if err != nil {
	89	return fmt.Errorf("cat-file: %s", strings.TrimSpace(header))
	90	}
	91	data := make([]byte, size+1) // the object and its trailing newline
	92	if _, err := io.ReadFull(r, data); err != nil {
	93	return err
	94	}
	95	if !fn(i, data[:size]) {
	96	return errStopped
	97	}
	98	}
	99	return nil
	100	}()
	101	if readErr != nil {
	102	cancel()
	103	}
	104	io.Copy(io.Discard, stdout)
	105	waitErr := cmd.Wait()
	106	switch {
	107	case readErr == errStopped:
	108	return nil
	109	case parent.Err() != nil:
	110	return parent.Err()
	111	case readErr != nil:
	112	return readErr
	113	}
	114	return waitErr
	115	}
	116
	117	var errStopped = errors.New("stopped")

internal/hookd/symbols_test.go added +63

		@@ -0,0 +1,63 @@
	1	package hookd
	2
	3	import (
	4	"context"
	5	"os"
	6	"path/filepath"
	7	"testing"
	8
	9	"gitbay.org/gitbay/internal/control"
	10	"gitbay.org/gitbay/internal/policy"
	11	"gitbay.org/gitbay/internal/store"
	12	"gitbay.org/gitbay/internal/symbols"
	13	)
	14
	15	// A push to the default branch asks for a symbol index and builds none
	16	// itself; a push to another branch asks for nothing. The worker builds
	17	// the index afterwards.
	18	func TestPostReceiveRequestsSymbolIndex(t *testing.T) {
	19	f := newShapeFixture(t)
	20	requested := func() bool {
	21	t.Helper()
	22	reqs, err := f.st.SymbolRequests()
	23	if err != nil {
	24	t.Fatal(err)
	25	}
	26	return len(reqs) == 1 && reqs[0].RepoID == f.repo.ID && !reqs[0].Force
	27	}
	28
	29	f.git(f.src, "checkout", "-q", "-b", "side")
	30	side := f.appCommit("side")
	31	f.sync()
	32	f.srv.postReceive(Request{RepoID: f.repo.ID, UserID: f.uid, Scope: "full", Updates: []policy.RefUpdate{
	33	{Ref: "refs/heads/side", Old: zeroSHA40, New: side}}})
	34	if requested() {
	35	t.Fatal("a push to a side branch asked for a symbol index")
	36	}
	37
	38	f.git(f.src, "checkout", "-q", "main")
	39	os.WriteFile(filepath.Join(f.src, "main.go"), []byte("package main\n\nfunc Serve() {}\n"), 0o644)
	40	f.git(f.src, "add", ".")
	41	f.git(f.src, "commit", "-q", "-m", "go")
	42	head := f.sha("HEAD")
	43	f.sync()
	44	f.srv.postReceive(Request{RepoID: f.repo.ID, UserID: f.uid, Scope: "full", Updates: []policy.RefUpdate{
	45	{Ref: "refs/heads/main", Old: f.base, New: head}}})
	46	if !requested() {
	47	t.Fatal("a push to the default branch did not ask for a symbol index")
	48	}
	49	if _, err := f.st.SymbolIndexFor(f.repo.ID); err != store.ErrNotFound {
	50	t.Fatalf("the push built an index itself: %v", err)
	51	}
	52
	53	w := symbols.New(f.st, func(owner, name string) string { return control.RepoDir(f.root, owner, name) })
	54	w.Sweep(context.Background())
	55	x, err := f.st.SymbolIndexFor(f.repo.ID)
	56	if err != nil \|\| x.Commit != head \|\| x.State != "ok" {
	57	t.Fatalf("index after the sweep = %+v, %v", x, err)
	58	}
	59	rows, _ := f.st.SearchSymbols(x.ID, "Serve", "", 0, 0)
	60	if len(rows) != 1 \|\| rows[0].Path != "main.go" \|\| rows[0].Line != 3 {
	61	t.Fatalf("Serve = %+v", rows)
	62	}
	63	}

internal/mirror/mirror.go +3

		@@ -84,6 +84,9 @@ func (w *Worker) sweep() {
84	84	w.St.SetMirrorResult(m.ID, err.Error())
85	85	} else {
86	86	w.St.SetMirrorResult(m.ID, "")
	87	if m.Direction == "pull" {
	88	w.St.RequestSymbolIndex(m.RepoID, false)
	89	}
87	90	}
88	91	}
89	92	}

internal/store/migrations/0072_symbols.down.sql added +3

		@@ -0,0 +1,3 @@
	1	DROP TABLE symbol_requests;
	2	DROP TABLE symbols;
	3	DROP TABLE symbol_indexes;

internal/store/migrations/0072_symbols.up.sql added +45

		@@ -0,0 +1,45 @@
	1	-- One symbol index per repository: the definitions found in the tree of
	2	-- the default branch's head (#293). tree is the key: a head whose tree is
	3	-- the indexed one needs no new index. state is ok, partial (a bound was
	4	-- reached; note says which) or failed (note says why), and a failed or
	5	-- partial index is kept as the record for its tree rather than retried.
	6	-- AUTOINCREMENT because a replaced index must not hand its id to the
	7	-- next: a paging cursor names the index it was taken from.
	8	CREATE TABLE symbol_indexes (
	9	id INTEGER PRIMARY KEY AUTOINCREMENT,
	10	repo_id INTEGER NOT NULL UNIQUE REFERENCES repos(id) ON DELETE CASCADE,
	11	commit_sha TEXT NOT NULL,
	12	tree TEXT NOT NULL,
	13	state TEXT NOT NULL CHECK (state IN ('ok', 'partial', 'failed')),
	14	note TEXT NOT NULL DEFAULT '',
	15	files INTEGER NOT NULL DEFAULT 0,
	16	symbols INTEGER NOT NULL DEFAULT 0,
	17	built_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
	18	);
	19	-- name is what is listed, key the name as written at a use site (they
	20	-- differ for Go methods: Type.Method and Method). lname and lkey are
	21	-- their lower-case forms, for case-insensitive prefix ranges.
	22	CREATE TABLE symbols (
	23	id INTEGER PRIMARY KEY,
	24	index_id INTEGER NOT NULL REFERENCES symbol_indexes(id) ON DELETE CASCADE,
	25	name TEXT NOT NULL,
	26	lname TEXT NOT NULL,
	27	key TEXT NOT NULL,
	28	lkey TEXT NOT NULL,
	29	kind TEXT NOT NULL,
	30	path TEXT NOT NULL,
	31	line INTEGER NOT NULL
	32	);
	33	CREATE INDEX symbols_lname ON symbols(index_id, lname);
	34	CREATE INDEX symbols_lkey ON symbols(index_id, lkey);
	35	CREATE INDEX symbols_key ON symbols(index_id, key);
	36	CREATE INDEX symbols_path ON symbols(index_id, path, line);
	37	-- Repositories waiting for the index worker. seq counts requests, so a
	38	-- push that lands while a build runs leaves its request in place; force
	39	-- rebuilds even when the tree is the indexed one.
	40	CREATE TABLE symbol_requests (
	41	repo_id INTEGER PRIMARY KEY REFERENCES repos(id) ON DELETE CASCADE,
	42	seq INTEGER NOT NULL DEFAULT 1,
	43	force INTEGER NOT NULL DEFAULT 0,
	44	requested_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
	45	);

internal/store/symbols.go added +251

		@@ -0,0 +1,251 @@
	1	package store
	2
	3	import (
	4	"database/sql"
	5	"errors"
	6	"strings"
	7	"unicode/utf8"
	8	)
	9
	10	// SymbolIndex is a repository's symbol index: what was indexed and how it
	11	// went. The states are described with the table's schema.
	12	type SymbolIndex struct {
	13	ID int64
	14	RepoID int64
	15	Commit string
	16	Tree string
	17	State string // ok \| partial \| failed
	18	Note string
	19	Files int
	20	Symbols int
	21	BuiltAt string
	22	}
	23
	24	// SymbolRow is one definition. Key is the name as written where it is
	25	// used; see internal/symbols.
	26	type SymbolRow struct {
	27	ID int64
	28	Name string
	29	Key string
	30	Kind string
	31	Path string
	32	Line int
	33	}
	34
	35	// SymbolTarget is where a key is defined: Count definitions, the first of
	36	// them at Path and Line.
	37	type SymbolTarget struct {
	38	Count int
	39	Path string
	40	Line int
	41	}
	42
	43	// SymbolRequest is a repository waiting for the index worker.
	44	type SymbolRequest struct {
	45	RepoID int64
	46	Seq int64
	47	Force bool
	48	}
	49
	50	// RequestSymbolIndex queues a repository for the index worker. A request
	51	// already waiting is bumped, so one taken by a running build is not
	52	// cleared when that build ends; force is kept once set.
	53	func (s *Store) RequestSymbolIndex(repoID int64, force bool) error {
	54	_, err := s.DB.Exec(`
	55	INSERT INTO symbol_requests (repo_id, force) VALUES (?, ?)
	56	ON CONFLICT (repo_id) DO UPDATE SET seq = seq + 1,
	57	force = MAX(force, excluded.force),
	58	requested_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')`, repoID, force)
	59	return err
	60	}
	61
	62	// SymbolRequests lists the waiting repositories, oldest request first.
	63	func (s *Store) SymbolRequests() ([]SymbolRequest, error) {
	64	rows, err := s.DB.Query(`SELECT repo_id, seq, force FROM symbol_requests ORDER BY requested_at, repo_id`)
	65	if err != nil {
	66	return nil, err
	67	}
	68	defer rows.Close()
	69	var out []SymbolRequest
	70	for rows.Next() {
	71	var r SymbolRequest
	72	if err := rows.Scan(&r.RepoID, &r.Seq, &r.Force); err != nil {
	73	return nil, err
	74	}
	75	out = append(out, r)
	76	}
	77	return out, rows.Err()
	78	}
	79
	80	// DoneSymbolRequest clears a request the worker has handled, unless it was
	81	// requested again since it was read.
	82	func (s *Store) DoneSymbolRequest(r SymbolRequest) error {
	83	_, err := s.DB.Exec("DELETE FROM symbol_requests WHERE repo_id = ? AND seq = ?", r.RepoID, r.Seq)
	84	return err
	85	}
	86
	87	// SymbolIndexFor returns a repository's index, or ErrNotFound when none
	88	// has been built.
	89	func (s *Store) SymbolIndexFor(repoID int64) (SymbolIndex, error) {
	90	var x SymbolIndex
	91	err := s.DB.QueryRow(`
	92	SELECT id, repo_id, commit_sha, tree, state, note, files, symbols, built_at
	93	FROM symbol_indexes WHERE repo_id = ?`, repoID).
	94	Scan(&x.ID, &x.RepoID, &x.Commit, &x.Tree, &x.State, &x.Note, &x.Files, &x.Symbols, &x.BuiltAt)
	95	if errors.Is(err, sql.ErrNoRows) {
	96	return x, ErrNotFound
	97	}
	98	return x, err
	99	}
	100
	101	// symbolInsertRows is how many rows one INSERT carries: eight columns
	102	// each, well under SQLite's variable limit.
	103	const symbolInsertRows = 200
	104
	105	// ReplaceSymbolIndex swaps a repository's index for a new one in one
	106	// transaction: readers see the old index or the new, never a mix. The
	107	// symbols count is taken from syms.
	108	func (s *Store) ReplaceSymbolIndex(x SymbolIndex, syms []SymbolRow) (int64, error) {
	109	tx, err := s.DB.Begin()
	110	if err != nil {
	111	return 0, err
	112	}
	113	defer tx.Rollback()
	114	if _, err := tx.Exec("DELETE FROM symbol_indexes WHERE repo_id = ?", x.RepoID); err != nil {
	115	return 0, err
	116	}
	117	res, err := tx.Exec(`
	118	INSERT INTO symbol_indexes (repo_id, commit_sha, tree, state, note, files, symbols)
	119	VALUES (?, ?, ?, ?, ?, ?, ?)`, x.RepoID, x.Commit, x.Tree, x.State, x.Note, x.Files, len(syms))
	120	if err != nil {
	121	return 0, err
	122	}
	123	id, err := res.LastInsertId()
	124	if err != nil {
	125	return 0, err
	126	}
	127	for len(syms) > 0 {
	128	n := min(len(syms), symbolInsertRows)
	129	var q strings.Builder
	130	q.WriteString("INSERT INTO symbols (index_id, name, lname, key, lkey, kind, path, line) VALUES ")
	131	args := make([]any, 0, n*8)
	132	for i, r := range syms[:n] {
	133	if i > 0 {
	134	q.WriteString(",")
	135	}
	136	q.WriteString("(?,?,?,?,?,?,?,?)")
	137	args = append(args, id, r.Name, strings.ToLower(r.Name), r.Key, strings.ToLower(r.Key), r.Kind, r.Path, r.Line)
	138	}
	139	if _, err := tx.Exec(q.String(), args...); err != nil {
	140	return 0, err
	141	}
	142	syms = syms[n:]
	143	}
	144	return id, tx.Commit()
	145	}
	146
	147	// SearchSymbols finds the symbols whose name or key starts with q, ignoring
	148	// case, ranked: an exact match first, then a prefix match, each
	149	// case-sensitive before case-insensitive, then by name, path and line.
	150	// after is the id of the last row of the previous page, 0 for the first;
	151	// an id that is no longer in the result gives an empty page. kind filters
	152	// when not empty.
	153	func (s *Store) SearchSymbols(indexID int64, q, kind string, limit int, after int64) ([]SymbolRow, error) {
	154	lq := strings.ToLower(q)
	155	hi := lq + "\U0010FFFF"
	156	n := utf8.RuneCountInString(q)
	157	lim := -1
	158	if limit > 0 {
	159	lim = limit
	160	}
	161	rows, err := s.DB.Query(`
	162	WITH m AS (
	163	SELECT id, name, key, kind, path, line,
	164	CASE WHEN name = ? OR key = ? THEN 0
	165	WHEN substr(name, 1, ?) = ? OR substr(key, 1, ?) = ? THEN 1
	166	WHEN lname = ? OR lkey = ? THEN 2
	167	ELSE 3 END AS rank
	168	FROM symbols
	169	WHERE index_id = ?
	170	AND ((lname >= ? AND lname < ?) OR (lkey >= ? AND lkey < ?))
	171	AND (? = '' OR kind = ?)
	172	)
	173	SELECT id, name, key, kind, path, line FROM m
	174	WHERE ? = 0 OR (rank, name, path, line, id) >
	175	(SELECT rank, name, path, line, id FROM m WHERE id = ?)
	176	ORDER BY rank, name, path, line, id
	177	LIMIT ?`,
	178	q, q, n, q, n, q, lq, lq,
	179	indexID, lq, hi, lq, hi, kind, kind,
	180	after, after, lim)
	181	if err != nil {
	182	return nil, err
	183	}
	184	return scanSymbols(rows)
	185	}
	186
	187	// SymbolsInFile lists one file's symbols in line order.
	188	func (s *Store) SymbolsInFile(indexID int64, path string) ([]SymbolRow, error) {
	189	rows, err := s.DB.Query(`
	190	SELECT id, name, key, kind, path, line FROM symbols
	191	WHERE index_id = ? AND path = ? ORDER BY line, id`, indexID, path)
	192	if err != nil {
	193	return nil, err
	194	}
	195	return scanSymbols(rows)
	196	}
	197
	198	func scanSymbols(rows *sql.Rows) ([]SymbolRow, error) {
	199	defer rows.Close()
	200	var out []SymbolRow
	201	for rows.Next() {
	202	var r SymbolRow
	203	if err := rows.Scan(&r.ID, &r.Name, &r.Key, &r.Kind, &r.Path, &r.Line); err != nil {
	204	return nil, err
	205	}
	206	out = append(out, r)
	207	}
	208	return out, rows.Err()
	209	}
	210
	211	// symbolKeysPerQuery bounds the IN list of one SymbolTargets query.
	212	const symbolKeysPerQuery = 500
	213
	214	// SymbolTargets looks up where each of keys is defined, case-sensitively.
	215	// A key with no definition is absent from the result.
	216	func (s *Store) SymbolTargets(indexID int64, keys []string) (map[string]SymbolTarget, error) {
	217	out := map[string]SymbolTarget{}
	218	for len(keys) > 0 {
	219	n := min(len(keys), symbolKeysPerQuery)
	220	args := make([]any, 0, n+1)
	221	args = append(args, indexID)
	222	for _, k := range keys[:n] {
	223	args = append(args, k)
	224	}
	225	// SQLite takes the bare columns of an aggregate query with MIN
	226	// from the row that holds the minimum: the first definition.
	227	rows, err := s.DB.Query(`
	228	SELECT key, COUNT(*), path, line, MIN(id) FROM symbols
	229	WHERE index_id = ? AND key IN (?`+strings.Repeat(",?", n-1)+`)
	230	GROUP BY key`, args...)
	231	if err != nil {
	232	return nil, err
	233	}
	234	for rows.Next() {
	235	var k string
	236	var t SymbolTarget
	237	var id int64
	238	if err := rows.Scan(&k, &t.Count, &t.Path, &t.Line, &id); err != nil {
	239	rows.Close()
	240	return nil, err
	241	}
	242	out[k] = t
	243	}
	244	rows.Close()
	245	if err := rows.Err(); err != nil {
	246	return nil, err
	247	}
	248	keys = keys[n:]
	249	}
	250	return out, nil
	251	}

internal/store/symbols_test.go added +93

		@@ -0,0 +1,93 @@
	1	package store
	2
	3	import (
	4	"strings"
	5	"testing"
	6	)
	7
	8	func symbolFixture(t testing.T) (Store, int64) {
	9	t.Helper()
	10	s := open(t)
	11	if err := s.MigrateUp(); err != nil {
	12	t.Fatal(err)
	13	}
	14	uid, err := s.CreateUser("alice", false)
	15	if err != nil {
	16	t.Fatal(err)
	17	}
	18	repoID, err := s.CreateRepo("user", uid, "app", "public")
	19	if err != nil {
	20	t.Fatal(err)
	21	}
	22	idx, err := s.ReplaceSymbolIndex(SymbolIndex{RepoID: repoID, Commit: "c", Tree: "t", State: "ok"}, []SymbolRow{
	23	{Name: "handlers", Key: "handlers", Kind: "var", Path: "b.go", Line: 1},
	24	{Name: "Server.Handle", Key: "Handle", Kind: "method", Path: "a.go", Line: 9},
	25	{Name: "Handle", Key: "Handle", Kind: "function", Path: "z.go", Line: 3},
	26	{Name: "handle", Key: "handle", Kind: "function", Path: "c.go", Line: 2},
	27	{Name: "HandleFunc", Key: "HandleFunc", Kind: "function", Path: "a.go", Line: 20},
	28	{Name: "Other", Key: "Other", Kind: "type", Path: "a.go", Line: 30},
	29	})
	30	if err != nil {
	31	t.Fatal(err)
	32	}
	33	return s, idx
	34	}
	35
	36	func symbolNames(rows []SymbolRow) string {
	37	var out []string
	38	for _, r := range rows {
	39	out = append(out, r.Name)
	40	}
	41	return strings.Join(out, " ")
	42	}
	43
	44	// Exact before prefix, case-sensitive before not, then by name; a method
	45	// matches on the name it is called by as well as Type.Method.
	46	func TestSearchSymbolsRanks(t *testing.T) {
	47	s, idx := symbolFixture(t)
	48	rows, err := s.SearchSymbols(idx, "Handle", "", 0, 0)
	49	if err != nil {
	50	t.Fatal(err)
	51	}
	52	if got, want := symbolNames(rows), "Handle Server.Handle HandleFunc handle handlers"; got != want {
	53	t.Fatalf("got %q, want %q", got, want)
	54	}
	55	rows, _ = s.SearchSymbols(idx, "Handle", "method", 0, 0)
	56	if got := symbolNames(rows); got != "Server.Handle" {
	57	t.Fatalf("kind filter: %q", got)
	58	}
	59	rows, _ = s.SearchSymbols(idx, "server.", "", 0, 0)
	60	if got := symbolNames(rows); got != "Server.Handle" {
	61	t.Fatalf("Type. prefix: %q", got)
	62	}
	63
	64	// Paging walks the same order without repeats.
	65	var paged []SymbolRow
	66	var after int64
	67	for {
	68	page, err := s.SearchSymbols(idx, "handle", "", 2, after)
	69	if err != nil {
	70	t.Fatal(err)
	71	}
	72	if len(page) == 0 {
	73	break
	74	}
	75	paged = append(paged, page...)
	76	after = page[len(page)-1].ID
	77	}
	78	all, _ := s.SearchSymbols(idx, "handle", "", 0, 0)
	79	if symbolNames(paged) != symbolNames(all) \|\| len(all) != 5 {
	80	t.Fatalf("paged %q, all %q", symbolNames(paged), symbolNames(all))
	81	}
	82	}
	83
	84	func TestSymbolTargets(t *testing.T) {
	85	s, idx := symbolFixture(t)
	86	got, err := s.SymbolTargets(idx, []string{"Handle", "Other", "missing"})
	87	if err != nil {
	88	t.Fatal(err)
	89	}
	90	if len(got) != 2 \|\| got["Handle"].Count != 2 \|\| got["Other"] != (SymbolTarget{1, "a.go", 30}) {
	91	t.Fatalf("targets = %+v", got)
	92	}
	93	}

internal/symbols/extract.go added +342

		@@ -0,0 +1,342 @@
	1	// Package symbols finds definitions in source files and keeps one index of
	2	// them per repository, for `repo symbols` and the blob view's links from a
	3	// name to where it is defined.
	4	//
	5	// The indexer is pure Go, with no external tagger. Go is parsed with
	6	// go/parser; the other languages are matched a line at a time against a
	7	// small table of anchored patterns, which finds the common definition
	8	// shapes and misses the unusual ones.
	9	//
	10	// Go function, method (named Type.Method), type, const, var
	11	// Swift function, class, struct, enum, interface (protocol), type (typealias)
	12	// Rust function, struct, enum, interface (trait), type, module, const, macro
	13	// Python function, method (an indented def), class
	14	// JavaScript, TS function, class, interface, type, enum, const
	15	// C and C++ headers function (prototype), struct, enum, class, type (typedef), macro (#define)
	16	// Shell function
	17	// Org, Markdown section (a heading)
	18	//
	19	// The index covers the tree of the default branch's head and is keyed by
	20	// that tree's id, so a push that leaves the tree as it was is not indexed
	21	// again. Worker builds it in the background after a push to the default
	22	// branch.
	23	package symbols
	24
	25	import (
	26	"bytes"
	27	"go/ast"
	28	"go/parser"
	29	"go/token"
	30	"path"
	31	"regexp"
	32	"strings"
	33	)
	34
	35	// Kinds are the symbol kinds the indexer records, in the order they are
	36	// listed.
	37	var Kinds = []string{
	38	"function", "method", "class", "struct", "enum", "interface",
	39	"type", "const", "var", "module", "macro", "section",
	40	}
	41
	42	// ValidKind reports whether k is one of Kinds.
	43	func ValidKind(k string) bool {
	44	for _, v := range Kinds {
	45	if v == k {
	46	return true
	47	}
	48	}
	49	return false
	50	}
	51
	52	// Symbol is one definition in one file. Name is what is listed; Key is the
	53	// name as it is written where the symbol is used, which differs from Name
	54	// only for Go methods (Name "Server.Handle", Key "Handle").
	55	type Symbol struct {
	56	Name string
	57	Key string
	58	Kind string
	59	Line int
	60	}
	61
	62	// MaxFileBytes is the largest file the indexer reads.
	63	const MaxFileBytes = 1 << 20
	64
	65	// Skip reports whether a file is left out of the index before it is read:
	66	// too large, vendored, generated by its name, or in a language the indexer
	67	// does not know.
	68	func Skip(p string, size int64) bool {
	69	if size > MaxFileBytes \|\| Language(p) == "" {
	70	return true
	71	}
	72	for _, seg := range strings.Split(path.Dir(p), "/") {
	73	if seg == "vendor" \|\| seg == "node_modules" {
	74	return true
	75	}
	76	}
	77	base := path.Base(p)
	78	return strings.HasSuffix(base, "_gen.go") \|\| strings.HasSuffix(base, ".pb.go") \|\|
	79	strings.HasSuffix(base, ".min.js")
	80	}
	81
	82	var languages = map[string]string{
	83	".go": "go",
	84	".swift": "swift",
	85	".rs": "rust",
	86	".py": "python",
	87	".js": "js",
	88	".mjs": "js",
	89	".cjs": "js",
	90	".jsx": "js",
	91	".ts": "js",
	92	".tsx": "js",
	93	".mts": "js",
	94	".h": "c",
	95	".hh": "c",
	96	".hpp": "c",
	97	".hxx": "c",
	98	".sh": "shell",
	99	".bash": "shell",
	100	".zsh": "shell",
	101	".org": "org",
	102	".md": "markdown",
	103	".markdown": "markdown",
	104	}
	105
	106	// Language names the indexer's language for a path, or "" when it has
	107	// none.
	108	func Language(p string) string {
	109	return languages[strings.ToLower(path.Ext(p))]
	110	}
	111
	112	// generatedGo is the marker `go generate` tools write, per the Go
	113	// convention for generated files.
	114	var generatedGo = regexp.MustCompile(`(?m)^// Code generated .* DO NOT EDIT\.$`)
	115
	116	// Extract returns the definitions in one file, in line order.
	117	func Extract(p string, data []byte) []Symbol {
	118	switch lang := Language(p); lang {
	119	case "":
	120	return nil
	121	case "go":
	122	return extractGo(data)
	123	case "org", "markdown":
	124	return extractHeadings(lang, data)
	125	default:
	126	return extractLines(rules[lang], data)
	127	}
	128	}
	129
	130	func extractGo(data []byte) []Symbol {
	131	head := data
	132	if len(head) > 4096 {
	133	head = head[:4096]
	134	}
	135	if generatedGo.Match(head) {
	136	return nil
	137	}
	138	fset := token.NewFileSet()
	139	// A file that does not parse still yields the declarations before the
	140	// error.
	141	f, _ := parser.ParseFile(fset, "", data, parser.SkipObjectResolution)
	142	if f == nil {
	143	return nil
	144	}
	145	var out []Symbol
	146	add := func(id *ast.Ident, name, kind string) {
	147	if id == nil \|\| id.Name == "_" {
	148	return
	149	}
	150	out = append(out, Symbol{Name: name, Key: id.Name, Kind: kind, Line: fset.Position(id.Pos()).Line})
	151	}
	152	for _, d := range f.Decls {
	153	switch d := d.(type) {
	154	case *ast.FuncDecl:
	155	if d.Recv == nil \|\| len(d.Recv.List) == 0 {
	156	add(d.Name, d.Name.Name, "function")
	157	continue
	158	}
	159	recv := receiverName(d.Recv.List[0].Type)
	160	if recv == "" {
	161	add(d.Name, d.Name.Name, "method")
	162	continue
	163	}
	164	add(d.Name, recv+"."+d.Name.Name, "method")
	165	case *ast.GenDecl:
	166	for _, spec := range d.Specs {
	167	switch s := spec.(type) {
	168	case *ast.TypeSpec:
	169	add(s.Name, s.Name.Name, "type")
	170	case *ast.ValueSpec:
	171	kind := "var"
	172	if d.Tok == token.CONST {
	173	kind = "const"
	174	}
	175	for _, n := range s.Names {
	176	add(n, n.Name, kind)
	177	}
	178	}
	179	}
	180	}
	181	}
	182	return out
	183	}
	184
	185	// receiverName is the type a method is declared on, without a pointer or
	186	// type parameters.
	187	func receiverName(e ast.Expr) string {
	188	for {
	189	switch t := e.(type) {
	190	case *ast.StarExpr:
	191	e = t.X
	192	case *ast.ParenExpr:
	193	e = t.X
	194	case *ast.IndexExpr:
	195	e = t.X
	196	case *ast.IndexListExpr:
	197	e = t.X
	198	case *ast.Ident:
	199	return t.Name
	200	default:
	201	return ""
	202	}
	203	}
	204	}
	205
	206	// rule is one line pattern: the first submatch is the name.
	207	type rule struct {
	208	re *regexp.Regexp
	209	kind string
	210	}
	211
	212	func r(kind, pattern string) rule { return rule{regexp.MustCompile(pattern), kind} }
	213
	214	// ident is a name in the languages the line rules cover.
	215	const ident = `([A-Za-z_$][A-Za-z0-9_$]*)`
	216
	217	var rules = map[string][]rule{
	218	"swift": {
	219	r("function", `^\s(?:@\w+\s+)(?:(?:public\|private\|fileprivate\|internal\|open\|static\|class\|final\|override\|mutating\|nonisolated\|async\|convenience\|required)\s+)*func\s+`+ident),
	220	r("class", `^\s(?:@\w+\s+)(?:(?:public\|private\|fileprivate\|internal\|open\|final)\s+)*(?:class\|actor)\s+`+ident),
	221	r("struct", `^\s(?:@\w+\s+)(?:(?:public\|private\|fileprivate\|internal)\s+)*struct\s+`+ident),
	222	r("enum", `^\s(?:@\w+\s+)(?:(?:public\|private\|fileprivate\|internal\|indirect)\s+)*enum\s+`+ident),
	223	r("interface", `^\s(?:@\w+\s+)(?:(?:public\|private\|fileprivate\|internal)\s+)*protocol\s+`+ident),
	224	r("type", `^\s(?:(?:public\|private\|fileprivate\|internal)\s+)typealias\s+`+ident),
	225	},
	226	"rust": {
	227	r("function", `^\s(?:pub(?:$[^)]$)?\s+)?(?:(?:const\|async\|unsafe\|extern(?:\s+"[^"]")?)\s+)fn\s+`+ident),
	228	r("struct", `^\s(?:pub(?:$[^)]$)?\s+)?struct\s+`+ident),
	229	r("enum", `^\s(?:pub(?:$[^)]$)?\s+)?enum\s+`+ident),
	230	r("interface", `^\s(?:pub(?:$[^)]$)?\s+)?(?:unsafe\s+)?trait\s+`+ident),
	231	r("type", `^\s(?:pub(?:$[^)]$)?\s+)?type\s+`+ident),
	232	r("module", `^\s(?:pub(?:$[^)]$)?\s+)?mod\s+`+ident),
	233	r("const", `^\s(?:pub(?:$[^)]$)?\s+)?(?:const\|static(?:\s+mut)?)\s+`+ident+`\s*:`),
	234	r("macro", `^\s*macro_rules!\s+`+ident),
	235	},
	236	"python": {
	237	r("function", `^(?:async\s+)?def\s+`+ident),
	238	r("method", `^\s+(?:async\s+)?def\s+`+ident),
	239	r("class", `^\s*class\s+`+ident),
	240	},
	241	"js": {
	242	r("function", `^\s(?:export\s+)?(?:default\s+)?(?:async\s+)?function\s\?\s`+ident),
	243	r("class", `^\s*(?:export\s+)?(?:default\s+)?(?:abstract\s+)?class\s+`+ident),
	244	r("interface", `^\s*(?:export\s+)?(?:declare\s+)?interface\s+`+ident),
	245	r("type", `^\s(?:export\s+)?(?:declare\s+)?type\s+`+ident+`\s(?:<[^=]>)?\s=`),
	246	r("enum", `^\s*(?:export\s+)?(?:declare\s+)?(?:const\s+)?enum\s+`+ident),
	247	r("const", `^(?:export\s+)?const\s+`+ident+`\s(?::[^=])?=`),
	248	},
	249	"c": {
	250	r("macro", `^\s#\sdefine\s+`+ident),
	251	r("struct", `^\s(?:typedef\s+)?struct\s+`+ident+`\s\{`),
	252	r("enum", `^\s(?:typedef\s+)?enum\s+(?:class\s+)?`+ident+`\s(?::[^{]*)?\{`),
	253	r("class", `^\sclass\s+`+ident+`\s(?::[^{;]*)?\{`),
	254	r("type", `^\stypedef\s+[^;(]?\b`+ident+`\s*;`),
	255	r("type", `^\s}\s`+ident+`\s*;`),
	256	r("function", `^[A-Za-z_][\w\s\&:<>,]?[\s\&]`+ident+`\s$[^;{]$\s(?:const\s*)?;`),
	257	},
	258	"shell": {
	259	r("function", `^\sfunction\s+([A-Za-z_][A-Za-z0-9_:.-])`),
	260	r("function", `^\s([A-Za-z_][A-Za-z0-9_:.-])\s\s(?:\{\|$)`),
	261	},
	262	}
	263
	264	// keywords are never names, though a rule can capture one: Swift's
	265	// `class var x` reads as a class named var, C's `if (x);` as a prototype.
	266	var keywords = map[string]bool{
	267	"return": true, "if": true, "while": true, "for": true, "switch": true,
	268	"sizeof": true, "else": true, "case": true, "do": true, "goto": true,
	269	"var": true, "let": true, "func": true, "static": true,
	270	}
	271
	272	func extractLines(rs []rule, data []byte) []Symbol {
	273	var out []Symbol
	274	line := 0
	275	for len(data) > 0 {
	276	line++
	277	var text []byte
	278	if i := bytes.IndexByte(data, '\n'); i >= 0 {
	279	text, data = data[:i], data[i+1:]
	280	} else {
	281	text, data = data, nil
	282	}
	283	if len(text) > 1000 {
	284	continue
	285	}
	286	for _, ru := range rs {
	287	m := ru.re.FindSubmatch(text)
	288	if m == nil {
	289	continue
	290	}
	291	name := string(m[1])
	292	if keywords[name] {
	293	continue
	294	}
	295	out = append(out, Symbol{Name: name, Key: name, Kind: ru.kind, Line: line})
	296	break
	297	}
	298	}
	299	return out
	300	}
	301
	302	var (
	303	mdHeading = regexp.MustCompile(`^#{1,6}\s+(.+?)\s#\s*$`)
	304	orgHeading = regexp.MustCompile(`^\+\s+(.+?)\s$`)
	305	)
	306
	307	// extractHeadings lists headings as sections, skipping what sits inside a
	308	// code block, where a line starting with # or * is code.
	309	func extractHeadings(lang string, data []byte) []Symbol {
	310	var out []Symbol
	311	inBlock := false
	312	for i, text := range strings.Split(string(data), "\n") {
	313	trimmed := strings.TrimSpace(text)
	314	if lang == "markdown" {
	315	if strings.HasPrefix(trimmed, "```") \|\| strings.HasPrefix(trimmed, "~~~") {
	316	inBlock = !inBlock
	317	continue
	318	}
	319	} else {
	320	lower := strings.ToLower(trimmed)
	321	if strings.HasPrefix(lower, "#+begin_") {
	322	inBlock = true
	323	continue
	324	}
	325	if strings.HasPrefix(lower, "#+end_") {
	326	inBlock = false
	327	continue
	328	}
	329	}
	330	if inBlock \|\| len(text) > 1000 {
	331	continue
	332	}
	333	re := mdHeading
	334	if lang == "org" {
	335	re = orgHeading
	336	}
	337	if m := re.FindStringSubmatch(text); m != nil {
	338	out = append(out, Symbol{Name: m[1], Key: m[1], Kind: "section", Line: i + 1})
	339	}
	340	}
	341	return out
	342	}

internal/symbols/extract_test.go added +187

		@@ -0,0 +1,187 @@
	1	package symbols
	2
	3	import (
	4	"fmt"
	5	"strings"
	6	"testing"
	7	)
	8
	9	// got renders symbols as "kind name key line", one per entry, key left
	10	// out when it equals the name.
	11	func got(syms []Symbol) []string {
	12	var out []string
	13	for _, s := range syms {
	14	if s.Key != s.Name {
	15	out = append(out, fmt.Sprintf("%s %s %s %d", s.Kind, s.Name, s.Key, s.Line))
	16	continue
	17	}
	18	out = append(out, fmt.Sprintf("%s %s %d", s.Kind, s.Name, s.Line))
	19	}
	20	return out
	21	}
	22
	23	func TestExtract(t *testing.T) {
	24	tests := []struct {
	25	path string
	26	src string
	27	want []string
	28	}{
	29	{"a.go", `package a
	30
	31	// Doc.
	32	func Top() {}
	33
	34	type Server struct{}
	35
	36	type (
	37	List[T any] struct{}
	38	ID int
	39	)
	40
	41	func (s *Server) Handle() {}
	42	func (l List[T]) Push(v T) {}
	43	func (List[T]) Len() int { return 0 }
	44
	45	const Max = 3
	46	const (
	47	A, B = 1, 2
	48	_ = 9
	49	)
	50
	51	var global int
	52	`, []string{
	53	"function Top 4", "type Server 6", "type List 9", "type ID 10",
	54	"method Server.Handle Handle 13", "method List.Push Push 14", "method List.Len Len 15",
	55	"const Max 17", "const A 19", "const B 19", "var global 23",
	56	}},
	57	{"broken.go", "package a\n\nfunc Before() {}\n\nfunc (\n", []string{"function Before 3"}},
	58	{"gen.go", "// Code generated by stringer; DO NOT EDIT.\n\npackage a\n\nfunc X() {}\n", nil},
	59	{"a.swift", `import Foundation
	60
	61	public final class Store {
	62	static func make() -> Store { Store() }
	63	class var shared: Store { make() }
	64	}
	65	struct Point { var x: Int }
	66	enum Mode { case a }
	67	protocol Drawable {}
	68	actor Cache {}
	69	typealias Handler = () -> Void
	70	@MainActor func onMain() {}
	71	`, []string{
	72	"class Store 3", "function make 4", "struct Point 7", "enum Mode 8",
	73	"interface Drawable 9", "class Cache 10", "type Handler 11", "function onMain 12",
	74	}},
	75	{"lib.rs", `pub fn open() {}
	76	pub(crate) async fn fetch() {}
	77	struct Inner;
	78	pub enum Kind { A }
	79	pub trait Store {}
	80	type Result<T> = std::result::Result<T, Error>;
	81	mod tests {}
	82	pub const LIMIT: usize = 3;
	83	static mut COUNT: u32 = 0;
	84	macro_rules! bail {}
	85	impl Store for Inner {}
	86	`, []string{
	87	"function open 1", "function fetch 2", "struct Inner 3", "enum Kind 4",
	88	"interface Store 5", "type Result 6", "module tests 7", "const LIMIT 8",
	89	"const COUNT 9", "macro bail 10",
	90	}},
	91	{"app.py", `import os
	92
	93	def main():
	94	pass
	95
	96	class Client:
	97	async def get(self):
	98	return helper()
	99
	100	async def run():
	101	pass
	102	`, []string{"function main 3", "class Client 6", "method get 7", "function run 10"}},
	103	{"web.ts", `export function render() {}
	104	export default async function load() {}
	105	function* gen() {}
	106	export class View {}
	107	export interface Props {}
	108	export type ID = string;
	109	export type Map<K, V> = Record<K, V>;
	110	export const enum Color { Red }
	111	export const LIMIT = 10;
	112	const handler = () => {};
	113	const inner = 1;
	114	`, []string{
	115	"function render 1", "function load 2", "function gen 3", "class View 4",
	116	"interface Props 5", "type ID 6", "type Map 7", "enum Color 8",
	117	"const LIMIT 9", "const handler 10",
	118	}},
	119	{"api.h", `#ifndef API_H
	120	#define API_H
	121	#define MAX_LEN 64
	122	struct buffer {
	123	int len;
	124	};
	125	typedef struct buffer buffer_t;
	126	typedef enum mode { MODE_A } mode_t;
	127	int buffer_open(const char *path);
	128	static inline void buffer_data(buffer_t b);
	129	typedef struct {
	130	int x;
	131	} point;
	132	#endif
	133	`, []string{
	134	"macro API_H 2", "macro MAX_LEN 3", "struct buffer 4", "type buffer_t 7",
	135	"enum mode 8", "function buffer_open 9", "function buffer_data 10", "type point 13",
	136	}},
	137	{"deploy.sh", `#!/bin/sh
	138	build() {
	139	go build ./...
	140	}
	141	function release {
	142	build
	143	}
	144	ship_it ()
	145	{
	146	release
	147	}
	148	`, []string{"function build 2", "function release 5", "function ship_it 8"}},
	149	{"README.md", "# Title\n\nText\n\n```sh\n# not a heading\n```\n\n## Usage ##\n", []string{"section Title 1", "section Usage 9"}},
	150	{"notes.org", "* Top\n#+begin_src sh\n* not a heading\n#+end_src\n** Child heading\n", []string{"section Top 1", "section Child heading 5"}},
	151	{"data.json", `{"a": 1}`, nil},
	152	}
	153	for _, tc := range tests {
	154	t.Run(tc.path, func(t *testing.T) {
	155	g := got(Extract(tc.path, []byte(tc.src)))
	156	if strings.Join(g, "\n") != strings.Join(tc.want, "\n") {
	157	t.Errorf("got:\n%s\nwant:\n%s", strings.Join(g, "\n"), strings.Join(tc.want, "\n"))
	158	}
	159	})
	160	}
	161	}
	162
	163	func TestSkip(t *testing.T) {
	164	tests := []struct {
	165	path string
	166	size int64
	167	skip bool
	168	}{
	169	{"main.go", 100, false},
	170	{"main.go", MaxFileBytes + 1, true},
	171	{"main.go", MaxFileBytes, false},
	172	{"vendor/x/y.go", 10, true},
	173	{"web/node_modules/a/index.js", 10, true},
	174	{"internal/vendors/y.go", 10, false},
	175	{"zz_gen.go", 10, true},
	176	{"api.pb.go", 10, true},
	177	{"static/app.min.js", 10, true},
	178	{"static/app.js", 10, false},
	179	{"image.png", 10, true},
	180	{"Makefile", 10, true},
	181	}
	182	for _, tc := range tests {
	183	if got := Skip(tc.path, tc.size); got != tc.skip {
	184	t.Errorf("Skip(%q, %d) = %v, want %v", tc.path, tc.size, got, tc.skip)
	185	}
	186	}
	187	}

internal/symbols/worker.go added +175

		@@ -0,0 +1,175 @@
	1	package symbols
	2
	3	import (
	4	"context"
	5	"errors"
	6	"fmt"
	7	"log/slog"
	8	"os"
	9	"time"
	10
	11	"gitbay.org/gitbay/internal/gitutil"
	12	"gitbay.org/gitbay/internal/store"
	13	)
	14
	15	// Bounds on one index run.
	16	const (
	17	// DefaultMaxSymbols is where a repository's index stops growing.
	18	DefaultMaxSymbols = 200_000
	19	// DefaultMaxTime is how long one run may take.
	20	DefaultMaxTime = 2 * time.Minute
	21	)
	22
	23	// Worker builds the index for each repository that has asked for one:
	24	// post-receive and the merge path ask after the default branch moves, and
	25	// `admin symbols reindex` asks with force. One repository at a time.
	26	type Worker struct {
	27	St *store.Store
	28	RepoDir func(owner, name string) string
	29	Tick time.Duration
	30	MaxSymbols int
	31	MaxTime time.Duration
	32	}
	33
	34	func New(st store.Store, repoDir func(owner, name string) string) Worker {
	35	tick := 5 * time.Second
	36	if v := os.Getenv("GITBAY_SYMBOLS_TICK"); v != "" {
	37	if d, err := time.ParseDuration(v); err == nil {
	38	tick = d
	39	}
	40	}
	41	return &Worker{St: st, RepoDir: repoDir, Tick: tick,
	42	MaxSymbols: DefaultMaxSymbols, MaxTime: DefaultMaxTime}
	43	}
	44
	45	// Run sweeps until ctx ends.
	46	func (w *Worker) Run(ctx context.Context) {
	47	t := time.NewTicker(w.Tick)
	48	defer t.Stop()
	49	for {
	50	select {
	51	case <-ctx.Done():
	52	return
	53	case <-t.C:
	54	w.Sweep(ctx)
	55	}
	56	}
	57	}
	58
	59	// Sweep handles every waiting request once. A request is cleared whatever
	60	// the outcome, so a repository whose index fails is not retried until
	61	// it is asked for again.
	62	func (w *Worker) Sweep(ctx context.Context) {
	63	reqs, err := w.St.SymbolRequests()
	64	if err != nil {
	65	slog.Error("symbols: listing requests", "err", err)
	66	return
	67	}
	68	for _, req := range reqs {
	69	if ctx.Err() != nil {
	70	return
	71	}
	72	err := w.Index(ctx, req.RepoID, req.Force)
	73	if ctx.Err() != nil {
	74	return // shutting down: the request stays for the next start
	75	}
	76	if err != nil {
	77	slog.Warn("symbols: indexing", "repo", req.RepoID, "err", err)
	78	}
	79	w.St.DoneSymbolRequest(req)
	80	}
	81	}
	82
	83	// Index brings one repository's index up to its default branch's head.
	84	// A head whose tree is already indexed is left alone unless force is set.
	85	// The error is for the log: an index that could not be built is recorded
	86	// as failed, and one cut short by a bound as partial.
	87	func (w *Worker) Index(ctx context.Context, repoID int64, force bool) error {
	88	repo, err := w.St.RepoByID(repoID)
	89	if errors.Is(err, store.ErrNotFound) {
	90	return nil
	91	} else if err != nil {
	92	return err
	93	}
	94	dir := w.RepoDir(repo.OwnerName, repo.Name)
	95	commit, err := gitutil.ResolveRef(dir, "refs/heads/"+repo.DefaultBranch)
	96	if err != nil {
	97	return nil // no default branch yet: nothing to index
	98	}
	99	tree, err := gitutil.ResolveTree(dir, commit)
	100	if err != nil {
	101	return err
	102	}
	103	if cur, err := w.St.SymbolIndexFor(repo.ID); err == nil && cur.Tree == tree && !force {
	104	return nil
	105	}
	106	x := store.SymbolIndex{RepoID: repo.ID, Commit: commit, Tree: tree, State: "ok"}
	107	syms, files, runErr := w.collect(ctx, dir, tree)
	108	if ctx.Err() != nil {
	109	return ctx.Err()
	110	}
	111	x.Files = files
	112	switch {
	113	case errors.Is(runErr, errSymbolCap):
	114	x.State, x.Note = "partial", fmt.Sprintf("stopped at %d symbols", w.MaxSymbols)
	115	case errors.Is(runErr, context.DeadlineExceeded):
	116	x.State, x.Note = "partial", fmt.Sprintf("stopped after %s", w.MaxTime)
	117	case runErr != nil:
	118	x.State, x.Note, syms = "failed", runErr.Error(), nil
	119	}
	120	if _, err := w.St.ReplaceSymbolIndex(x, syms); err != nil {
	121	return err
	122	}
	123	if x.State != "ok" {
	124	return fmt.Errorf("%s %s: %s", repo.Path(), x.State, x.Note)
	125	}
	126	return nil
	127	}
	128
	129	var errSymbolCap = errors.New("symbol cap reached")
	130
	131	// collect reads every indexable blob in tree and extracts its symbols,
	132	// stopping at the symbol cap or the time bound with what it has.
	133	func (w *Worker) collect(ctx context.Context, dir, tree string) ([]store.SymbolRow, int, error) {
	134	ctx, cancel := context.WithTimeout(ctx, w.MaxTime)
	135	defer cancel()
	136	blobs, err := gitutil.ListBlobs(ctx, dir, tree)
	137	if err != nil {
	138	if ctx.Err() != nil {
	139	return nil, 0, ctx.Err()
	140	}
	141	return nil, 0, err
	142	}
	143	var paths, shas []string
	144	for _, b := range blobs {
	145	if b.Mode == "120000" \|\| b.Mode == "160000" \|\| Skip(b.Name, b.Size) {
	146	continue
	147	}
	148	paths = append(paths, b.Name)
	149	shas = append(shas, b.SHA)
	150	}
	151	var out []store.SymbolRow
	152	files := 0
	153	capped := false
	154	err = gitutil.CatBlobs(ctx, dir, shas, func(i int, data []byte) bool {
	155	if gitutil.IsBinary(data) {
	156	return true
	157	}
	158	files++
	159	for _, s := range Extract(paths[i], data) {
	160	if len(out) >= w.MaxSymbols {
	161	capped = true
	162	return false
	163	}
	164	out = append(out, store.SymbolRow{Name: s.Name, Key: s.Key, Kind: s.Kind, Path: paths[i], Line: s.Line})
	165	}
	166	return true
	167	})
	168	if capped {
	169	return out, files, errSymbolCap
	170	}
	171	if err != nil && ctx.Err() == nil {
	172	return nil, files, err
	173	}
	174	return out, files, err
	175	}

internal/symbols/worker_test.go added +234

		@@ -0,0 +1,234 @@
	1	package symbols
	2
	3	import (
	4	"context"
	5	"os"
	6	"os/exec"
	7	"path/filepath"
	8	"strings"
	9	"testing"
	10	"time"
	11
	12	"gitbay.org/gitbay/internal/store"
	13	)
	14
	15	type fixture struct {
	16	t *testing.T
	17	st *store.Store
	18	repo store.Repo
	19	src string
	20	bare string
	21	w *Worker
	22	}
	23
	24	func newFixture(t testing.T) fixture {
	25	t.Helper()
	26	st, err := store.Open(":memory:")
	27	if err != nil {
	28	t.Fatal(err)
	29	}
	30	t.Cleanup(func() { st.Close() })
	31	if err := st.MigrateUp(); err != nil {
	32	t.Fatal(err)
	33	}
	34	uid, err := st.CreateUser("alice", false)
	35	if err != nil {
	36	t.Fatal(err)
	37	}
	38	id, err := st.CreateRepo("user", uid, "app", "public")
	39	if err != nil {
	40	t.Fatal(err)
	41	}
	42	repo, err := st.RepoByID(id)
	43	if err != nil {
	44	t.Fatal(err)
	45	}
	46	root := t.TempDir()
	47	f := &fixture{t: t, st: st, repo: repo, src: filepath.Join(root, "src"), bare: filepath.Join(root, "app.git")}
	48	f.w = &Worker{St: st, RepoDir: func(owner, name string) string { return f.bare },
	49	MaxSymbols: DefaultMaxSymbols, MaxTime: DefaultMaxTime}
	50	f.git(root, "init", "-q", "-b", repo.DefaultBranch, "src")
	51	f.git(root, "init", "-q", "--bare", f.bare)
	52	return f
	53	}
	54
	55	func (f *fixture) git(dir string, args ...string) string {
	56	f.t.Helper()
	57	cmd := exec.Command("git", args...)
	58	cmd.Dir = dir
	59	cmd.Env = append(os.Environ(),
	60	"GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null",
	61	"GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.test",
	62	"GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.test")
	63	out, err := cmd.CombinedOutput()
	64	if err != nil {
	65	f.t.Fatalf("git %v: %v\n%s", args, err, out)
	66	}
	67	return strings.TrimSpace(string(out))
	68	}
	69
	70	// commit writes files (an empty content removes one) and pushes the
	71	// default branch to the bare repository.
	72	func (f *fixture) commit(files map[string]string) string {
	73	f.t.Helper()
	74	for name, content := range files {
	75	p := filepath.Join(f.src, name)
	76	if content == "" {
	77	os.Remove(p)
	78	continue
	79	}
	80	os.MkdirAll(filepath.Dir(p), 0o755)
	81	if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
	82	f.t.Fatal(err)
	83	}
	84	}
	85	f.git(f.src, "add", "-A")
	86	f.git(f.src, "commit", "-q", "--allow-empty", "-m", "c")
	87	f.git(f.src, "push", "-q", "--force", f.bare, "HEAD:refs/heads/"+f.repo.DefaultBranch)
	88	return f.git(f.src, "rev-parse", "HEAD")
	89	}
	90
	91	func (f *fixture) sweep(force bool) store.SymbolIndex {
	92	f.t.Helper()
	93	if err := f.st.RequestSymbolIndex(f.repo.ID, force); err != nil {
	94	f.t.Fatal(err)
	95	}
	96	f.w.Sweep(context.Background())
	97	if reqs, _ := f.st.SymbolRequests(); len(reqs) != 0 {
	98	f.t.Fatalf("request left after a sweep: %+v", reqs)
	99	}
	100	x, err := f.st.SymbolIndexFor(f.repo.ID)
	101	if err != nil {
	102	f.t.Fatal(err)
	103	}
	104	return x
	105	}
	106
	107	func (f *fixture) names(x store.SymbolIndex) []string {
	108	f.t.Helper()
	109	rows, err := f.st.SearchSymbols(x.ID, "", "", 0, 0)
	110	if err != nil {
	111	f.t.Fatal(err)
	112	}
	113	var out []string
	114	for _, r := range rows {
	115	out = append(out, r.Path+":"+r.Name)
	116	}
	117	return out
	118	}
	119
	120	func TestWorkerIndexesDefaultBranch(t *testing.T) {
	121	f := newFixture(t)
	122	head := f.commit(map[string]string{
	123	"main.go": "package main\n\nfunc Hello() {}\n",
	124	"vendor/dep/dep.go": "package dep\n\nfunc Vendored() {}\n",
	125	"gen/zz_gen.go": "package gen\n\nfunc Generated() {}\n",
	126	"big/big.go": "package big\n\nfunc Big() {}\n" + strings.Repeat("//\n", MaxFileBytes/3),
	127	"docs/guide.md": "# Guide\n",
	128	"assets/logo.png": "\x89PNG\r\n",
	129	"node_modules/x.js": "function hidden() {}\n",
	130	"web/app.js": "function shown() {}\n",
	131	})
	132	x := f.sweep(false)
	133	if x.State != "ok" \|\| x.Commit != head {
	134	t.Fatalf("index = %+v, want ok at %s", x, head)
	135	}
	136	got := strings.Join(f.names(x), " ")
	137	for _, want := range []string{"main.go:Hello", "docs/guide.md:Guide", "web/app.js:shown"} {
	138	if !strings.Contains(got, want) {
	139	t.Errorf("missing %s in %s", want, got)
	140	}
	141	}
	142	for _, skip := range []string{"Vendored", "Generated", "Big", "hidden"} {
	143	if strings.Contains(got, skip) {
	144	t.Errorf("%s indexed: %s", skip, got)
	145	}
	146	}
	147	if x.Files != 3 \|\| x.Symbols != 3 {
	148	t.Errorf("files %d symbols %d, want 3 and 3", x.Files, x.Symbols)
	149	}
	150	}
	151
	152	// A new commit whose tree is the indexed one is not indexed again; a
	153	// changed tree replaces the index, and force rebuilds an unchanged one.
	154	func TestWorkerKeysOnTree(t *testing.T) {
	155	f := newFixture(t)
	156	f.commit(map[string]string{"a.go": "package a\n\nfunc One() {}\n"})
	157	first := f.sweep(false)
	158
	159	f.commit(nil) // same tree, new commit
	160	same := f.sweep(false)
	161	if same.ID != first.ID \|\| same.Commit != first.Commit {
	162	t.Fatalf("unchanged tree was reindexed: %+v then %+v", first, same)
	163	}
	164
	165	f.commit(map[string]string{"a.go": "package a\n\nfunc Two() {}\n"})
	166	changed := f.sweep(false)
	167	if changed.ID == first.ID \|\| changed.Tree == first.Tree {
	168	t.Fatalf("changed tree kept the old index: %+v", changed)
	169	}
	170	if got := f.names(changed); len(got) != 1 \|\| got[0] != "a.go:Two" {
	171	t.Fatalf("symbols = %v, want only the new tree's", got)
	172	}
	173	var n int
	174	f.st.DB.QueryRow("SELECT COUNT(*) FROM symbol_indexes WHERE repo_id = ?", f.repo.ID).Scan(&n)
	175	if n != 1 {
	176	t.Fatalf("%d indexes for one repository, want 1", n)
	177	}
	178
	179	forced := f.sweep(true)
	180	if forced.ID == changed.ID \|\| forced.Tree != changed.Tree {
	181	t.Fatalf("force did not rebuild: %+v then %+v", changed, forced)
	182	}
	183	}
	184
	185	func TestWorkerSymbolCap(t *testing.T) {
	186	f := newFixture(t)
	187	f.commit(map[string]string{"a.go": "package a\n\nfunc A() {}\nfunc B() {}\nfunc C() {}\n"})
	188	f.w.MaxSymbols = 2
	189	x := f.sweep(false)
	190	if x.State != "partial" \|\| x.Symbols != 2 \|\| !strings.Contains(x.Note, "2 symbols") {
	191	t.Fatalf("index = %+v, want partial with 2 symbols", x)
	192	}
	193	}
	194
	195	func TestWorkerTimeBound(t *testing.T) {
	196	f := newFixture(t)
	197	f.commit(map[string]string{"a.go": "package a\n\nfunc A() {}\n"})
	198	f.w.MaxTime = time.Nanosecond
	199	x := f.sweep(false)
	200	if x.State != "partial" \|\| !strings.Contains(x.Note, "stopped after") {
	201	t.Fatalf("index = %+v, want partial on the time bound", x)
	202	}
	203	}
	204
	205	// A tree that cannot be read is recorded as failed, and the record stands
	206	// for that tree: asking again does not retry it.
	207	func TestWorkerFailureIsRecordedNotRetried(t *testing.T) {
	208	f := newFixture(t)
	209	f.commit(map[string]string{"a.go": "package a\n\nfunc A() {}\n"})
	210	blob := f.git(f.bare, "rev-parse", "refs/heads/"+f.repo.DefaultBranch+":a.go")
	211	if err := os.Remove(filepath.Join(f.bare, "objects", blob[:2], blob[2:])); err != nil {
	212	t.Fatal(err)
	213	}
	214	x := f.sweep(false)
	215	if x.State != "failed" \|\| x.Note == "" \|\| x.Symbols != 0 {
	216	t.Fatalf("index = %+v, want failed with a note", x)
	217	}
	218	again := f.sweep(false)
	219	if again.ID != x.ID {
	220	t.Fatalf("a failed tree was retried: %+v then %+v", x, again)
	221	}
	222	}
	223
	224	func TestWorkerIgnoresEmptyRepository(t *testing.T) {
	225	f := newFixture(t)
	226	f.st.RequestSymbolIndex(f.repo.ID, false)
	227	f.w.Sweep(context.Background())
	228	if _, err := f.st.SymbolIndexFor(f.repo.ID); err != store.ErrNotFound {
	229	t.Fatalf("an empty repository has an index: %v", err)
	230	}
	231	if reqs, _ := f.st.SymbolRequests(); len(reqs) != 0 {
	232	t.Fatalf("request left: %+v", reqs)
	233	}
	234	}

Commit b1cab2086b

Commit `b1cab2086b`