Commit `c528009533`

c52800953399a64c7a80d7970a8772dfe17873f8

parent: 7a781d1ccb

Verified · cmc ci/build: success ci/test: success

cmc <hello@cleberg.net> · 2026-09-29 05:56 UTC

symbols: chunked builds, failure records, name and byte bounds

A new index is written in 5000-row transactions under a building id no
read sees, published in one short transaction, and the replaced one is
deleted in chunks; the next run removes an interrupted build. A failed
run is recorded beside the current index, which stays in use, and is
retried once after an hour or on the next push. Names over 256 bytes
are dropped and a run stops at 32 MiB of names and paths. The blob
view resolves its ref once and passes the commit to IndexedTree; repo
symbols and the results page take queries of two characters or more,
and the page's cursor names its index.

Closes #293

Layout: unified · split

.gitbay/wiki/Users.org +14 −8

		@@ -252,7 +252,8 @@ gitbay repo unarchive you/project # writes refused, browsing
252	252	A push to the default branch queues a symbol index of its tree, built in
253	253	the background after the push completes; a tree that is already indexed
254	254	is not indexed again. =repo symbols <owner/name> [--kind k] <query>=
255		lists definitions whose name starts with the query, exact matches
	255	lists definitions whose name starts with the query (two characters
	256	at least), exact matches
256	257	first, case-sensitive before case-insensitive, as name, kind, path and
257	258	line; =--limit= and =--cursor= page it, and without them it stops at
258	259	200 rows. A Go method is listed as =Type.Method= and also matches on
		@@ -276,13 +277,18 @@ What is indexed:
276	277	Languages other than Go are matched a line at a time, so unusual
277	278	definition shapes are missed. Files over 1 MiB, anything under
278	279	=vendor/= or =node_modules/=, =_gen.go=, =.pb.go=, Go files marked
279		=Code generated ... DO NOT EDIT.=, and =*.min.js= are skipped. One run
280		stops at 200,000 symbols or two minutes and records the index as
281		partial; an index that cannot be built is recorded as failed. Either
282		record stands for its tree: a push that changes the tree builds again,
283		one that does not is left alone. A repository that has not been pushed
284		to since the index existed has none until its next push to the default
285		branch. =admin symbols reindex <owner/name>= rebuilds one regardless.
	280	=Code generated ... DO NOT EDIT.=, and =*.min.js= are skipped, and a
	281	name longer than 256 bytes is dropped. One run stops at 200,000
	282	symbols, 32 MiB of names and paths, or two minutes, and publishes what
	283	it found as a partial index that says which bound it reached.
	284
	285	A new index is written while the previous one stays in use, and
	286	replaces it in one step. A run that cannot build an index records why
	287	and leaves the previous index current; the same tree is tried again
	288	once after an hour, or when a push changes the tree. A repository that
	289	has not been pushed to since the index existed has none until its next
	290	push to the default branch. =admin symbols reindex <owner/name>=
	291	rebuilds one regardless.
286	292
287	293	On the web, a file viewed at the indexed tree (the default branch's
288	294	head, or any commit with the same tree) links each name the index

internal/control/symbols.go +31 −20

		@@ -59,8 +59,8 @@ func runRepoSymbols(c *Ctx, args []string) int {
59	59	if path == "" \|\| query == "" {
60	60	return c.usage()
61	61	}
62		if len(query) > maxQueryLen {
63		return c.fail(protocol.ExitUsage, "query must be 1 to %d characters", maxQueryLen)
	62	if len(query) < MinSymbolQuery \|\| len(query) > maxQueryLen {
	63	return c.fail(protocol.ExitUsage, "query must be %d to %d characters", MinSymbolQuery, maxQueryLen)
64	64	}
65	65	if kind != "" && !symbols.ValidKind(kind) {
66	66	return c.fail(protocol.ExitUsage, "--kind must be one of %s", strings.Join(symbols.Kinds, ", "))
		@@ -71,27 +71,28 @@ func runRepoSymbols(c *Ctx, args []string) int {
71	71	}
72	72	idx, err := c.Store.SymbolIndexFor(repo.ID)
73	73	if errors.Is(err, store.ErrNotFound) {
	74	if f, ferr := c.Store.SymbolFailureFor(repo.ID); ferr == nil {
	75	return c.fail(protocol.ExitFailure, "the symbol index of %s could not be built: %s", repo.Path(), f.Note)
	76	}
74	77	return c.fail(protocol.ExitNotFound, "%s has no symbol index yet; one is built after a push to %s", repo.Path(), repo.DefaultBranch)
75	78	} else if err != nil {
76	79	return c.fail(protocol.ExitFailure, "%v", err)
77	80	}
78	81	if ref != "" && ref != repo.DefaultBranch {
79	82	dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
80		if !IndexedTree(dir, ref, idx) {
	83	sha, err := gitutil.ResolveRef(dir, ref)
	84	if err != nil \|\| !IndexedTree(dir, sha, idx) {
81	85	return c.fail(protocol.ExitNotFound, "only the default branch, %s, is indexed; %s is not at the indexed tree", repo.DefaultBranch, ref)
82	86	}
83	87	}
84		if idx.State == "failed" {
85		return c.fail(protocol.ExitFailure, "the symbol index of %s failed: %s", repo.Path(), idx.Note)
86		}
87	88	var after int64
88	89	if p.key != "" {
89		cursorIdx, id, ok := parseSymbolCursor(p.key)
	90	cursorIdx, id, ok := ParseSymbolCursor(p.key)
90	91	if !ok {
91	92	return c.fail(protocol.ExitUsage, "bad cursor")
92	93	}
93	94	if cursorIdx != idx.ID {
94		return c.fail(protocol.ExitUsage, "the symbol index was rebuilt since that cursor; start again without --cursor")
	95	return c.fail(protocol.ExitUsage, "%s", StaleSymbolCursor)
95	96	}
96	97	after = id
97	98	}
		@@ -108,7 +109,7 @@ func runRepoSymbols(c *Ctx, args []string) int {
108	109	rows = rows[:symbolsUnpaged]
109	110	}
110	111	rows, next := trimPage(p, rows, "symbol", func(r store.SymbolRow) string {
111		return strconv.FormatInt(idx.ID, 10) + "." + strconv.FormatInt(r.ID, 10)
	112	return SymbolCursor(idx.ID, r.ID)
112	113	})
113	114	var ds []symbolOut
114	115	for _, r := range rows {
		@@ -129,10 +130,24 @@ func runRepoSymbols(c *Ctx, args []string) int {
129	130	})
130	131	}
131	132
132		// parseSymbolCursor reads "<index id>.<row id>". The index id makes a
	133	// MinSymbolQuery is the shortest query repo symbols and the results page
	134	// take: one character matches too much of an index to be worth a page.
	135	const MinSymbolQuery = 2
	136
	137	// StaleSymbolCursor is the refusal for a cursor taken from an index that
	138	// has since been replaced.
	139	const StaleSymbolCursor = "the symbol index was rebuilt since that cursor; start again without it"
	140
	141	// SymbolCursor is the paging key after row id of index indexID, the form
	142	// ParseSymbolCursor reads.
	143	func SymbolCursor(indexID, id int64) string {
	144	return strconv.FormatInt(indexID, 10) + "." + strconv.FormatInt(id, 10)
	145	}
	146
	147	// ParseSymbolCursor reads "<index id>.<row id>". The index id makes a
133	148	// cursor from before a rebuild fail rather than page through the new
134	149	// index from an unrelated row.
135		func parseSymbolCursor(key string) (int64, int64, bool) {
	150	func ParseSymbolCursor(key string) (int64, int64, bool) {
136	151	a, b, ok := strings.Cut(key, ".")
137	152	if !ok {
138	153	return 0, 0, false
		@@ -142,15 +157,11 @@ func parseSymbolCursor(key string) (int64, int64, bool) {
142	157	return idx, id, err1 == nil && err2 == nil && id > 0
143	158	}
144	159
145		// IndexedTree reports whether ref names a commit whose tree is the one
146		// idx was built from: the default branch's head when the index is
147		// current, or any other commit with the same content. The blob view uses
148		// it to decide whether its names can link into the index.
149		func IndexedTree(dir, ref string, idx store.SymbolIndex) bool {
150		sha, err := gitutil.ResolveRef(dir, ref)
151		if err != nil {
152		return false
153		}
	160	// IndexedTree reports whether commit sha has the tree idx was built from:
	161	// the default branch's head when the index is current, or any other
	162	// commit with the same content. The blob view uses it to decide whether
	163	// its names can link into the index.
	164	func IndexedTree(dir, sha string, idx store.SymbolIndex) bool {
154	165	if sha == idx.Commit {
155	166	return true
156	167	}

internal/control/symbols_test.go +1

		@@ -114,6 +114,7 @@ func TestRepoSymbolsRefusals(t *testing.T) {
114	114	{"another ref", alice, []string{"repo", "symbols", repo.Path(), "--ref", "feature", "Parse"}, protocol.ExitNotFound, "only the default branch"},
115	115	{"unknown kind", alice, []string{"repo", "symbols", repo.Path(), "--kind", "widget", "Parse"}, protocol.ExitUsage, "--kind"},
116	116	{"no query", alice, []string{"repo", "symbols", repo.Path()}, protocol.ExitUsage, ""},
	117	{"one-character query", alice, []string{"repo", "symbols", repo.Path(), "P"}, protocol.ExitUsage, "2 to"},
117	118	{"reindex needs an admin", alice, []string{"admin", "symbols", "reindex", repo.Path()}, protocol.ExitDenied, "admin"},
118	119	} {
119	120	t.Run(tc.name, func(t *testing.T) {

internal/httpd/symbols.go +25 −12

		@@ -2,6 +2,7 @@ package httpd
2	2
3	3	import (
4	4	"errors"
	5	"fmt"
5	6	"html/template"
6	7	"net/http"
7	8	"net/url"
		@@ -29,14 +30,14 @@ func escapePath(p string) string {
29	30	}
30	31
31	32	// blobSymbols finds the viewed file's symbols and links the names in its
32		// highlighted source to their definitions, when the ref being viewed has
33		// the indexed tree. A name defined once links to that line at the same
	33	// highlighted source to their definitions, when commit, the ref being
	34	// viewed as the page resolved it, has the indexed tree. A name defined once links to that line at the same
34	35	// ref; one defined more than once links to the results page. Names the
35	36	// index does not hold stay plain. The whole page costs one lookup of the
36	37	// file's distinct names, however many times each appears.
37		func (s *Server) blobSymbols(p repoPage, filePath string, code template.HTML) (template.HTML, []store.SymbolRow) {
	38	func (s *Server) blobSymbols(p repoPage, commit, filePath string, code template.HTML) (template.HTML, []store.SymbolRow) {
38	39	idx, err := s.st.SymbolIndexFor(p.Repo.ID)
39		if err != nil \|\| idx.State == "failed" \|\| !control.IndexedTree(p.Dir, p.Ref, idx) {
	40	if err != nil \|\| !control.IndexedTree(p.Dir, commit, idx) {
40	41	return code, nil
41	42	}
42	43	list, _ := s.st.SymbolsInFile(idx.ID, filePath)
		@@ -101,20 +102,32 @@ func (s Server) symbolsPage(w http.ResponseWriter, r http.Request) {
101	102	switch {
102	103	case errors.Is(err, store.ErrNotFound):
103	104	problem = "no symbol index yet; one is built after a push to " + p.Repo.DefaultBranch
	105	if f, ferr := s.st.SymbolFailureFor(p.Repo.ID); ferr == nil {
	106	problem = "the symbol index could not be built: " + f.Note
	107	}
104	108	case err != nil:
105	109	http.Error(w, "internal error", http.StatusInternalServerError)
106	110	return
107		case idx.State == "failed":
108		problem = "the symbol index failed: " + idx.Note
109		case len(q) > 200:
110		problem = "query must be 1 to 200 characters"
111		case q != "":
	111	case q == "":
	112	case len(q) < control.MinSymbolQuery \|\| len(q) > 200:
	113	problem = fmt.Sprintf("query must be %d to 200 characters", control.MinSymbolQuery)
	114	default:
112	115	if idx.State == "partial" {
113	116	note = "the index is partial: " + idx.Note
114	117	}
115	118	var after int64
116		if a, err := strconv.ParseInt(r.URL.Query().Get("after"), 10, 64); err == nil && a > 0 {
117		after = a
	119	if a := r.URL.Query().Get("after"); a != "" {
	120	cursorIdx, id, ok := control.ParseSymbolCursor(a)
	121	switch {
	122	case !ok:
	123	problem = "bad cursor"
	124	case cursorIdx != idx.ID:
	125	problem = control.StaleSymbolCursor
	126	}
	127	after = id
	128	}
	129	if problem != "" {
	130	break
118	131	}
119	132	rows, err = s.st.SearchSymbols(idx.ID, q, kind, symbolsPageSize+1, after)
120	133	if err != nil {
		@@ -123,7 +136,7 @@ func (s Server) symbolsPage(w http.ResponseWriter, r http.Request) {
123	136	}
124	137	if len(rows) > symbolsPageSize {
125	138	rows = rows[:symbolsPageSize]
126		v := url.Values{"q": {q}, "after": {strconv.FormatInt(rows[len(rows)-1].ID, 10)}}
	139	v := url.Values{"q": {q}, "after": {control.SymbolCursor(idx.ID, rows[len(rows)-1].ID)}}
127	140	if kind != "" {
128	141	v.Set("kind", kind)
129	142	}

internal/httpd/symbols_test.go +10 −3

		@@ -62,8 +62,7 @@ func symbolServer(t testing.T) Server {
62	62	symbolGit(t, src, "add", ".")
63	63	symbolGit(t, src, "commit", "-q", "-m", "side")
64	64
65		w := &symbols.Worker{St: st, RepoDir: func(owner, name string) string { return control.RepoDir(root, owner, name) },
66		MaxSymbols: symbols.DefaultMaxSymbols, MaxTime: symbols.DefaultMaxTime}
	65	w := symbols.NewWith(st, func(owner, name string) string { return control.RepoDir(root, owner, name) }, 0)
67	66	for _, r := range []struct{ name, vis string }{{"app", "public"}, {"secret", "private"}} {
68	67	id, err := st.CreateRepo("user", uid, r.name, r.vis)
69	68	if err != nil {
		@@ -73,7 +72,7 @@ func symbolServer(t testing.T) Server {
73	72	os.MkdirAll(filepath.Dir(dir), 0o755)
74	73	symbolGit(t, root, "init", "-q", "--bare", dir)
75	74	symbolGit(t, src, "push", "-q", dir, "main", "side")
76		if err := w.Index(context.Background(), id, false); err != nil {
	75	if _, err := w.Index(context.Background(), id, false); err != nil {
77	76	t.Fatal(err)
78	77	}
79	78	}
		@@ -124,6 +123,14 @@ func TestSymbolsPage(t *testing.T) {
124	123	if strings.Contains(w.Body.String(), "a/dup.go:3") {
125	124	t.Error("kind filter ignored")
126	125	}
	126	if w := get(t, h, "/alice/app/symbols?q=d", nil); !strings.Contains(w.Body.String(), "query must be 2 to 200 characters") {
	127	t.Error("a one-character query was searched")
	128	}
	129	// A cursor from an index that has been replaced says so rather than
	130	// showing an empty page.
	131	if w := get(t, h, "/alice/app/symbols?q=dup&after=999999.1", nil); !strings.Contains(w.Body.String(), "rebuilt since that cursor") {
	132	t.Errorf("stale cursor:\n%s", w.Body.String())
	133	}
127	134	for _, p := range []string{"/alice/secret/symbols?q=dup", "/alice/secret/blob/main/main.go"} {
128	135	if w := get(t, h, p, nil); w.Code != 404 {
129	136	t.Errorf("%s anonymously: %d, want 404", p, w.Code)

internal/httpd/web.go +9 −2

		@@ -694,7 +694,14 @@ func (s Server) blob(w http.ResponseWriter, r http.Request) {
694	694	}
695	695	p.Tab = "files"
696	696	filePath := strings.Trim(r.PathValue("path"), "/")
697		data, err := gitutil.ReadBlob(p.Dir, p.Ref, filePath, maxRenderBytes+1)
	697	// The file and its symbol links are read from one commit, even if the
	698	// ref moves while the page renders.
	699	commit, err := gitutil.ResolveRef(p.Dir, p.Ref)
	700	if err != nil {
	701	s.notFound(w, r)
	702	return
	703	}
	704	data, err := gitutil.ReadBlob(p.Dir, commit, filePath, maxRenderBytes+1)
698	705	if err != nil {
699	706	s.notFound(w, r)
700	707	return
		@@ -708,7 +715,7 @@ func (s Server) blob(w http.ResponseWriter, r http.Request) {
708	715	}
709	716	var fileSymbols []store.SymbolRow
710	717	if !binary && !image {
711		codeHTML, fileSymbols = s.blobSymbols(p, filePath, codeHTML)
	718	codeHTML, fileSymbols = s.blobSymbols(p, commit, filePath, codeHTML)
712	719	}
713	720	// Markdown and org render like a README, with the source one click
714	721	// away; ?view=source shows the text instead.

internal/store/migrations/0072_symbols.down.sql +1

		@@ -1,3 +1,4 @@
1	1	DROP TABLE symbol_requests;
	2	DROP TABLE symbol_failures;
2	3	DROP TABLE symbols;
3	4	DROP TABLE symbol_indexes;

internal/store/migrations/0072_symbols.up.sql +27 −2

		@@ -1,21 +1,29 @@
	1	-- Symbol indexes (#293): the definitions found in the tree of a
	2	-- repository's default branch head. tree is the key: a head whose tree
	3	-- is the current index's needs no new one.
	4	--
	5	-- An index is written in chunks while its state is building, which no
	6	-- read sees; publishing it makes it ok or partial (a bound was reached,
	7	-- note says which) and retires the previous one in one short
	8	-- transaction. Building and retired rows are deleted, symbols first and
	9	-- in chunks, by the worker. At most one index per repository is current.
	10	--
1	11	-- AUTOINCREMENT because a replaced index must not hand its id to the
2	12	-- next: a paging cursor names the index it was taken from.
3	13	CREATE TABLE symbol_indexes (
4	14	id INTEGER PRIMARY KEY AUTOINCREMENT,
5		repo_id INTEGER NOT NULL UNIQUE REFERENCES repos(id) ON DELETE CASCADE,
	15	repo_id INTEGER NOT NULL REFERENCES repos(id) ON DELETE CASCADE,
6	16	commit_sha TEXT NOT NULL,
7	17	tree TEXT NOT NULL,
8		state TEXT NOT NULL CHECK (state IN ('ok', 'partial', 'failed')),
	18	state TEXT NOT NULL CHECK (state IN ('building', 'ok', 'partial', 'retired')),
9	19	note TEXT NOT NULL DEFAULT '',
10	20	files INTEGER NOT NULL DEFAULT 0,
11	21	symbols INTEGER NOT NULL DEFAULT 0,
12	22	built_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
13	23	);
	24	CREATE INDEX symbol_indexes_repo ON symbol_indexes(repo_id, state);
	25	CREATE UNIQUE INDEX symbol_indexes_current ON symbol_indexes(repo_id)
	26	WHERE state IN ('ok', 'partial');
14	27	-- name is what is listed, key the name as written at a use site (they
15	28	-- differ for Go methods: Type.Method and Method). lname and lkey are
16	29	-- their lower-case forms, for case-insensitive prefix ranges.
		@@ -34,12 +42,23 @@ CREATE INDEX symbols_lname ON symbols(index_id, lname);
34	42	CREATE INDEX symbols_lkey ON symbols(index_id, lkey);
35	43	CREATE INDEX symbols_key ON symbols(index_id, key);
36	44	CREATE INDEX symbols_path ON symbols(index_id, path, line);
	45	-- The last run that could not build an index, kept apart so the current
	46	-- index stays in place. Cleared by the next index published.
	47	CREATE TABLE symbol_failures (
	48	repo_id INTEGER PRIMARY KEY REFERENCES repos(id) ON DELETE CASCADE,
	49	tree TEXT NOT NULL,
	50	note TEXT NOT NULL,
	51	failed_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
	52	);
37	53	-- Repositories waiting for the index worker. seq counts requests, so a
38	54	-- push that lands while a build runs leaves its request in place; force
	55	-- rebuilds even when the tree is the indexed one. not_before defers a
	56	-- retry after a failure; attempts counts those retries.
39	57	CREATE TABLE symbol_requests (
40	58	repo_id INTEGER PRIMARY KEY REFERENCES repos(id) ON DELETE CASCADE,
41	59	seq INTEGER NOT NULL DEFAULT 1,
42	60	force INTEGER NOT NULL DEFAULT 0,
	61	attempts INTEGER NOT NULL DEFAULT 0,
	62	not_before TEXT NOT NULL DEFAULT '',
43	63	requested_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
44	64	);

internal/store/symbols.go +199 −38

		@@ -3,24 +3,32 @@ package store
3	3	import (
4	4	"database/sql"
5	5	"errors"
	6	"fmt"
6	7	"strings"
7	8	"unicode/utf8"
8	9	)
9	10
10		// SymbolIndex is a repository's symbol index: what was indexed and how it
11		// went. The states are described with the table's schema.
	11	// SymbolIndex is one symbol index of a repository. The states are
	12	// described with the table's schema; reads only ever see ok or partial.
12	13	type SymbolIndex struct {
13	14	ID int64
14	15	RepoID int64
15	16	Commit string
16	17	Tree string
17		State string // ok \| partial \| failed
	18	State string // building \| ok \| partial \| retired
18	19	Note string
19	20	Files int
20	21	Symbols int
21	22	BuiltAt string
22	23	}
23	24
	25	// SymbolFailure is the last run that could not build an index.
	26	type SymbolFailure struct {
	27	Tree string
	28	Note string
	29	FailedAt string
	30	}
	31
24	32	// SymbolRow is one definition. Key is the name as written where it is
25	33	// used; see internal/symbols.
26	34	type SymbolRow struct {
		@@ -42,26 +50,31 @@ type SymbolTarget struct {
42	50
43	51	// SymbolRequest is a repository waiting for the index worker.
44	52	type SymbolRequest struct {
45		RepoID int64
46		Seq int64
47		Force bool
	53	RepoID int64
	54	Seq int64
	55	Force bool
	56	Attempts int
48	57	}
49	58
50	59	// RequestSymbolIndex queues a repository for the index worker. A request
51	60	// already waiting is bumped, so one taken by a running build is not
52		// cleared when that build ends; force is kept once set.
	61	// cleared when that build ends, and a deferred retry becomes due now;
	62	// force is kept once set.
53	63	func (s *Store) RequestSymbolIndex(repoID int64, force bool) error {
54	64	_, err := s.DB.Exec(`
55	65	INSERT INTO symbol_requests (repo_id, force) VALUES (?, ?)
56	66	ON CONFLICT (repo_id) DO UPDATE SET seq = seq + 1,
57		force = MAX(force, excluded.force),
	67	force = MAX(force, excluded.force), attempts = 0, not_before = '',
58	68	requested_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')`, repoID, force)
59	69	return err
60	70	}
61	71
62		// SymbolRequests lists the waiting repositories, oldest request first.
	72	// SymbolRequests lists the requests that are due, oldest first.
63	73	func (s *Store) SymbolRequests() ([]SymbolRequest, error) {
64		rows, err := s.DB.Query(`SELECT repo_id, seq, force FROM symbol_requests ORDER BY requested_at, repo_id`)
	74	rows, err := s.DB.Query(`
	75	SELECT repo_id, seq, force, attempts FROM symbol_requests
	76	WHERE not_before = '' OR not_before <= strftime('%Y-%m-%dT%H:%M:%fZ','now')
	77	ORDER BY requested_at, repo_id`)
65	78	if err != nil {
66	79	return nil, err
67	80	}
		@@ -69,7 +82,7 @@ func (s *Store) SymbolRequests() ([]SymbolRequest, error) {
69	82	var out []SymbolRequest
70	83	for rows.Next() {
71	84	var r SymbolRequest
72		if err := rows.Scan(&r.RepoID, &r.Seq, &r.Force); err != nil {
	85	if err := rows.Scan(&r.RepoID, &r.Seq, &r.Force, &r.Attempts); err != nil {
73	86	return nil, err
74	87	}
75	88	out = append(out, r)
		@@ -84,46 +97,65 @@ func (s *Store) DoneSymbolRequest(r SymbolRequest) error {
84	97	return err
85	98	}
86	99
87		// SymbolIndexFor returns a repository's index, or ErrNotFound when none
88		// has been built.
89		func (s *Store) SymbolIndexFor(repoID int64) (SymbolIndex, error) {
	100	// DeferSymbolRequest keeps a request for another attempt after seconds,
	101	// unless it was requested again since it was read.
	102	func (s *Store) DeferSymbolRequest(r SymbolRequest, seconds int) error {
	103	_, err := s.DB.Exec(`
	104	UPDATE symbol_requests SET force = 0, attempts = attempts + 1,
	105	not_before = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', ?)
	106	WHERE repo_id = ? AND seq = ?`, fmt.Sprintf("+%d seconds", seconds), r.RepoID, r.Seq)
	107	return err
	108	}
	109
	110	const symbolIndexCols = `id, repo_id, commit_sha, tree, state, note, files, symbols, built_at`
	111
	112	func scanSymbolIndex(row *sql.Row) (SymbolIndex, error) {
90	113	var x SymbolIndex
91		err := s.DB.QueryRow(`
92		SELECT id, repo_id, commit_sha, tree, state, note, files, symbols, built_at
93		FROM symbol_indexes WHERE repo_id = ?`, repoID).
94		Scan(&x.ID, &x.RepoID, &x.Commit, &x.Tree, &x.State, &x.Note, &x.Files, &x.Symbols, &x.BuiltAt)
	114	err := row.Scan(&x.ID, &x.RepoID, &x.Commit, &x.Tree, &x.State, &x.Note, &x.Files, &x.Symbols, &x.BuiltAt)
95	115	if errors.Is(err, sql.ErrNoRows) {
96	116	return x, ErrNotFound
97	117	}
98	118	return x, err
99	119	}
100	120
	121	// SymbolIndexFor returns a repository's current index, or ErrNotFound
	122	// when none has been published.
	123	func (s *Store) SymbolIndexFor(repoID int64) (SymbolIndex, error) {
	124	return scanSymbolIndex(s.DB.QueryRow(`SELECT `+symbolIndexCols+`
	125	FROM symbol_indexes WHERE repo_id = ? AND state IN ('ok', 'partial')`, repoID))
	126	}
	127
	128	// SymbolIndexByID returns an index in any state.
	129	func (s *Store) SymbolIndexByID(id int64) (SymbolIndex, error) {
	130	return scanSymbolIndex(s.DB.QueryRow(`SELECT `+symbolIndexCols+`
	131	FROM symbol_indexes WHERE id = ?`, id))
	132	}
	133
	134	// BeginSymbolIndex creates an index in the building state, which no read
	135	// sees until PublishSymbolIndex.
	136	func (s *Store) BeginSymbolIndex(repoID int64, commit, tree string) (int64, error) {
	137	res, err := s.DB.Exec(`
	138	INSERT INTO symbol_indexes (repo_id, commit_sha, tree, state)
	139	VALUES (?, ?, ?, 'building')`, repoID, commit, tree)
	140	if err != nil {
	141	return 0, err
	142	}
	143	return res.LastInsertId()
	144	}
	145
101	146	// symbolInsertRows is how many rows one INSERT carries: eight columns
102	147	// each, well under SQLite's variable limit.
103	148	const symbolInsertRows = 200
104	149
105		// ReplaceSymbolIndex swaps a repository's index for a new one in one
106		// transaction: readers see the old index or the new, never a mix. The
107		// symbols count is taken from syms.
108		func (s *Store) ReplaceSymbolIndex(x SymbolIndex, syms []SymbolRow) (int64, error) {
	150	// AddSymbols writes rows into an index being built, in one transaction.
	151	// The caller keeps each call to a few thousand rows so the write lock is
	152	// held briefly.
	153	func (s *Store) AddSymbols(indexID int64, syms []SymbolRow) error {
109	154	tx, err := s.DB.Begin()
110	155	if err != nil {
111		return 0, err
	156	return err
112	157	}
113	158	defer tx.Rollback()
114		if _, err := tx.Exec("DELETE FROM symbol_indexes WHERE repo_id = ?", x.RepoID); err != nil {
115		return 0, err
116		}
117		res, err := tx.Exec(`
118		INSERT INTO symbol_indexes (repo_id, commit_sha, tree, state, note, files, symbols)
119		VALUES (?, ?, ?, ?, ?, ?, ?)`, x.RepoID, x.Commit, x.Tree, x.State, x.Note, x.Files, len(syms))
120		if err != nil {
121		return 0, err
122		}
123		id, err := res.LastInsertId()
124		if err != nil {
125		return 0, err
126		}
127	159	for len(syms) > 0 {
128	160	n := min(len(syms), symbolInsertRows)
129	161	var q strings.Builder
		@@ -134,14 +166,143 @@ func (s *Store) ReplaceSymbolIndex(x SymbolIndex, syms []SymbolRow) (int64, erro
134	166	q.WriteString(",")
135	167	}
136	168	q.WriteString("(?,?,?,?,?,?,?,?)")
137		args = append(args, id, r.Name, strings.ToLower(r.Name), r.Key, strings.ToLower(r.Key), r.Kind, r.Path, r.Line)
	169	args = append(args, indexID, r.Name, strings.ToLower(r.Name), r.Key, strings.ToLower(r.Key), r.Kind, r.Path, r.Line)
138	170	}
139	171	if _, err := tx.Exec(q.String(), args...); err != nil {
	172	return err
	173	}
	174	syms = syms[n:]
	175	}
	176	return tx.Commit()
	177	}
	178
	179	// PublishSymbolIndex makes a built index the repository's current one, as
	180	// ok or partial, and retires the one it replaces, in one short
	181	// transaction. A recorded failure is cleared.
	182	func (s *Store) PublishSymbolIndex(x SymbolIndex) error {
	183	tx, err := s.DB.Begin()
	184	if err != nil {
	185	return err
	186	}
	187	defer tx.Rollback()
	188	if _, err := tx.Exec(`
	189	UPDATE symbol_indexes SET state = 'retired'
	190	WHERE repo_id = ? AND state IN ('ok', 'partial')`, x.RepoID); err != nil {
	191	return err
	192	}
	193	res, err := tx.Exec(`
	194	UPDATE symbol_indexes SET state = ?, note = ?, files = ?,
	195	symbols = (SELECT COUNT(*) FROM symbols WHERE index_id = ?),
	196	built_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')
	197	WHERE id = ? AND repo_id = ? AND state = 'building'`,
	198	x.State, x.Note, x.Files, x.ID, x.ID, x.RepoID)
	199	if err != nil {
	200	return err
	201	}
	202	if n, _ := res.RowsAffected(); n != 1 {
	203	return fmt.Errorf("symbol index %d is not being built", x.ID)
	204	}
	205	if _, err := tx.Exec("DELETE FROM symbol_failures WHERE repo_id = ?", x.RepoID); err != nil {
	206	return err
	207	}
	208	return tx.Commit()
	209	}
	210
	211	// symbolDeleteRows is how many symbols one purge transaction deletes.
	212	const symbolDeleteRows = 5000
	213
	214	// PurgeSymbolIndexes deletes a repository's indexes that are not current:
	215	// retired ones, and building ones left by a run that did not finish. The
	216	// symbols go a chunk per transaction, then the index row.
	217	func (s *Store) PurgeSymbolIndexes(repoID int64) error {
	218	rows, err := s.DB.Query(`SELECT id FROM symbol_indexes
	219	WHERE repo_id = ? AND state IN ('building', 'retired')`, repoID)
	220	if err != nil {
	221	return err
	222	}
	223	var ids []int64
	224	for rows.Next() {
	225	var id int64
	226	if err := rows.Scan(&id); err != nil {
	227	rows.Close()
	228	return err
	229	}
	230	ids = append(ids, id)
	231	}
	232	rows.Close()
	233	if err := rows.Err(); err != nil {
	234	return err
	235	}
	236	for _, id := range ids {
	237	for {
	238	res, err := s.DB.Exec(`DELETE FROM symbols WHERE id IN
	239	(SELECT id FROM symbols WHERE index_id = ? LIMIT ?)`, id, symbolDeleteRows)
	240	if err != nil {
	241	return err
	242	}
	243	if n, _ := res.RowsAffected(); n == 0 {
	244	break
	245	}
	246	}
	247	if _, err := s.DB.Exec(`DELETE FROM symbol_indexes
	248	WHERE id = ? AND state IN ('building', 'retired')`, id); err != nil {
	249	return err
	250	}
	251	}
	252	return nil
	253	}
	254
	255	// ReplaceSymbolIndex builds, publishes and purges in one call: the
	256	// worker's sequence, for a caller holding every row already.
	257	func (s *Store) ReplaceSymbolIndex(x SymbolIndex, syms []SymbolRow) (int64, error) {
	258	id, err := s.BeginSymbolIndex(x.RepoID, x.Commit, x.Tree)
	259	if err != nil {
	260	return 0, err
	261	}
	262	for len(syms) > 0 {
	263	n := min(len(syms), 5000)
	264	if err := s.AddSymbols(id, syms[:n]); err != nil {
140	265	return 0, err
141	266	}
142	267	syms = syms[n:]
143	268	}
144		return id, tx.Commit()
	269	x.ID = id
	270	if err := s.PublishSymbolIndex(x); err != nil {
	271	return 0, err
	272	}
	273	return id, s.PurgeSymbolIndexes(x.RepoID)
	274	}
	275
	276	// RecordSymbolFailure records a run that could not build an index. The
	277	// current index, if any, stays current.
	278	func (s *Store) RecordSymbolFailure(repoID int64, tree, note string) error {
	279	_, err := s.DB.Exec(`
	280	INSERT INTO symbol_failures (repo_id, tree, note) VALUES (?, ?, ?)
	281	ON CONFLICT (repo_id) DO UPDATE SET tree = excluded.tree, note = excluded.note,
	282	failed_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')`, repoID, tree, note)
	283	return err
	284	}
	285
	286	// SymbolFailureFor returns the last failure, or ErrNotFound.
	287	func (s *Store) SymbolFailureFor(repoID int64) (SymbolFailure, error) {
	288	var f SymbolFailure
	289	err := s.DB.QueryRow(`SELECT tree, note, failed_at FROM symbol_failures WHERE repo_id = ?`, repoID).
	290	Scan(&f.Tree, &f.Note, &f.FailedAt)
	291	if errors.Is(err, sql.ErrNoRows) {
	292	return f, ErrNotFound
	293	}
	294	return f, err
	295	}
	296
	297	// SymbolFailureRecent reports whether tree failed within the last
	298	// seconds.
	299	func (s *Store) SymbolFailureRecent(repoID int64, tree string, seconds int) (bool, error) {
	300	var n int
	301	err := s.DB.QueryRow(`SELECT COUNT(*) FROM symbol_failures
	302	WHERE repo_id = ? AND tree = ?
	303	AND failed_at > strftime('%Y-%m-%dT%H:%M:%fZ', 'now', ?)`,
	304	repoID, tree, fmt.Sprintf("-%d seconds", seconds)).Scan(&n)
	305	return n > 0, err
145	306	}
146	307
147	308	// SearchSymbols finds the symbols whose name or key starts with q, ignoring

internal/symbols/extract.go +16 −3

		@@ -113,18 +113,31 @@ func Language(p string) string {
113	113	// convention for generated files.
114	114	var generatedGo = regexp.MustCompile(`(?m)^// Code generated .* DO NOT EDIT\.$`)
115	115
	116	// MaxNameBytes is the longest name the index keeps. A longer one is
	117	// dropped: no real definition is named that way, and a file of them is a
	118	// way to fill the index.
	119	const MaxNameBytes = 256
	120
116	121	// Extract returns the definitions in one file, in line order.
117	122	func Extract(p string, data []byte) []Symbol {
	123	var syms []Symbol
118	124	switch lang := Language(p); lang {
119	125	case "":
120	126	return nil
121	127	case "go":
122		return extractGo(data)
	128	syms = extractGo(data)
123	129	case "org", "markdown":
124		return extractHeadings(lang, data)
	130	syms = extractHeadings(lang, data)
125	131	default:
126		return extractLines(rules[lang], data)
	132	syms = extractLines(rules[lang], data)
	133	}
	134	kept := syms[:0]
	135	for _, s := range syms {
	136	if len(s.Name) <= MaxNameBytes && len(s.Key) <= MaxNameBytes {
	137	kept = append(kept, s)
	138	}
127	139	}
	140	return kept
128	141	}
129	142
130	143	func extractGo(data []byte) []Symbol {

internal/symbols/worker.go +101 −29

		@@ -12,12 +12,20 @@ import (
12	12	"gitbay.org/gitbay/internal/store"
13	13	)
14	14
15		// Bounds on one index run.
	15	// Bounds on one index run, and how it writes.
16	16	const (
17	17	// DefaultMaxSymbols is where a repository's index stops growing.
18	18	DefaultMaxSymbols = 200_000
	19	// DefaultMaxBytes bounds the names, keys and paths one index holds, so
	20	// a tree of long names cannot fill the database within the count.
	21	DefaultMaxBytes = 32 << 20
19	22	// DefaultMaxTime is how long one run may take.
20	23	DefaultMaxTime = 2 * time.Minute
	24	// DefaultChunkRows is how many symbols one write transaction carries.
	25	DefaultChunkRows = 5000
	26	// DefaultBackoff is how long a failed tree waits before it is tried
	27	// again.
	28	DefaultBackoff = time.Hour
21	29	)
22	30
23	31	// Worker builds the index for each repository that has asked for one:
		@@ -28,7 +36,13 @@ type Worker struct {
28	36	RepoDir func(owner, name string) string
29	37	Tick time.Duration
30	38	MaxSymbols int
	39	MaxBytes int
31	40	MaxTime time.Duration
	41	ChunkRows int
	42	Backoff time.Duration
	43	// chunkHook runs after each chunk is written; tests read the index
	44	// mid-build through it.
	45	chunkHook func(indexID int64)
32	46	}
33	47
34	48	func New(st store.Store, repoDir func(owner, name string) string) Worker {
		@@ -38,8 +52,14 @@ func New(st store.Store, repoDir func(owner, name string) string) Worker {
38	52	tick = d
39	53	}
40	54	}
	55	return NewWith(st, repoDir, tick)
	56	}
	57
	58	// NewWith is New with the tick given, and every bound at its default.
	59	func NewWith(st store.Store, repoDir func(owner, name string) string, tick time.Duration) Worker {
41	60	return &Worker{St: st, RepoDir: repoDir, Tick: tick,
42		MaxSymbols: DefaultMaxSymbols, MaxTime: DefaultMaxTime}
	61	MaxSymbols: DefaultMaxSymbols, MaxBytes: DefaultMaxBytes, MaxTime: DefaultMaxTime,
	62	ChunkRows: DefaultChunkRows, Backoff: DefaultBackoff}
43	63	}
44	64
45	65	// Run sweeps until ctx ends.
		@@ -56,9 +76,10 @@ func (w *Worker) Run(ctx context.Context) {
56	76	}
57	77	}
58	78
59		// Sweep handles every waiting request once. A request is cleared whatever
60		// the outcome, so a repository whose index fails is not retried until
61		// it is asked for again.
	79	// Sweep handles every due request once. A request is cleared whatever
	80	// the outcome, except that a first failure is kept for one retry after
	81	// Backoff: a failure is retried once, not in a loop, and again only when
	82	// something asks.
62	83	func (w *Worker) Sweep(ctx context.Context) {
63	84	reqs, err := w.St.SymbolRequests()
64	85	if err != nil {
		@@ -69,67 +90,111 @@ func (w *Worker) Sweep(ctx context.Context) {
69	90	if ctx.Err() != nil {
70	91	return
71	92	}
72		err := w.Index(ctx, req.RepoID, req.Force)
	93	failed, err := w.Index(ctx, req.RepoID, req.Force)
73	94	if ctx.Err() != nil {
74	95	return // shutting down: the request stays for the next start
75	96	}
76	97	if err != nil {
77	98	slog.Warn("symbols: indexing", "repo", req.RepoID, "err", err)
78	99	}
	100	if failed && req.Attempts == 0 {
	101	w.St.DeferSymbolRequest(req, int(w.Backoff.Seconds()))
	102	continue
	103	}
79	104	w.St.DoneSymbolRequest(req)
80	105	}
81	106	}
82	107
83	108	// Index brings one repository's index up to its default branch's head.
84		// A head whose tree is already indexed is left alone unless force is set.
85		// The error is for the log: an index that could not be built is recorded
86		// as failed, and one cut short by a bound as partial.
87		func (w *Worker) Index(ctx context.Context, repoID int64, force bool) error {
	109	// A head whose tree is already indexed is left alone unless force is set,
	110	// and so is one whose tree failed within Backoff. The new index is
	111	// written in chunks while no read can see it, then published in one
	112	// short transaction. A run cut short by a bound publishes what it found
	113	// as partial; one that cannot build records the failure and leaves the
	114	// current index in place, reporting failed. The error is for the log.
	115	func (w *Worker) Index(ctx context.Context, repoID int64, force bool) (failed bool, err error) {
88	116	repo, err := w.St.RepoByID(repoID)
89	117	if errors.Is(err, store.ErrNotFound) {
90		return nil
	118	return false, nil
91	119	} else if err != nil {
92		return err
	120	return false, err
	121	}
	122	// A run that stopped part way left its building index behind.
	123	if err := w.St.PurgeSymbolIndexes(repo.ID); err != nil {
	124	return false, err
93	125	}
94	126	dir := w.RepoDir(repo.OwnerName, repo.Name)
95	127	commit, err := gitutil.ResolveRef(dir, "refs/heads/"+repo.DefaultBranch)
96	128	if err != nil {
97		return nil // no default branch yet: nothing to index
	129	return false, nil // no default branch yet: nothing to index
98	130	}
99	131	tree, err := gitutil.ResolveTree(dir, commit)
100	132	if err != nil {
101		return err
	133	return false, err
102	134	}
103		if cur, err := w.St.SymbolIndexFor(repo.ID); err == nil && cur.Tree == tree && !force {
104		return nil
	135	if !force {
	136	if cur, err := w.St.SymbolIndexFor(repo.ID); err == nil && cur.Tree == tree {
	137	return false, nil
	138	}
	139	if recent, err := w.St.SymbolFailureRecent(repo.ID, tree, int(w.Backoff.Seconds())); err != nil \|\| recent {
	140	return false, err
	141	}
105	142	}
106	143	x := store.SymbolIndex{RepoID: repo.ID, Commit: commit, Tree: tree, State: "ok"}
107	144	syms, files, runErr := w.collect(ctx, dir, tree)
108	145	if ctx.Err() != nil {
109		return ctx.Err()
	146	return false, ctx.Err()
110	147	}
111	148	x.Files = files
112	149	switch {
113	150	case errors.Is(runErr, errSymbolCap):
114	151	x.State, x.Note = "partial", fmt.Sprintf("stopped at %d symbols", w.MaxSymbols)
	152	case errors.Is(runErr, errByteBudget):
	153	x.State, x.Note = "partial", fmt.Sprintf("stopped at %d bytes of names and paths", w.MaxBytes)
115	154	case errors.Is(runErr, context.DeadlineExceeded):
116	155	x.State, x.Note = "partial", fmt.Sprintf("stopped after %s", w.MaxTime)
117	156	case runErr != nil:
118		x.State, x.Note, syms = "failed", runErr.Error(), nil
	157	if err := w.St.RecordSymbolFailure(repo.ID, tree, runErr.Error()); err != nil {
	158	return true, err
	159	}
	160	return true, fmt.Errorf("%s failed: %v", repo.Path(), runErr)
	161	}
	162	if x.ID, err = w.St.BeginSymbolIndex(repo.ID, commit, tree); err != nil {
	163	return false, err
119	164	}
120		if _, err := w.St.ReplaceSymbolIndex(x, syms); err != nil {
121		return err
	165	for len(syms) > 0 {
	166	if ctx.Err() != nil {
	167	return false, ctx.Err() // the next run purges what was written
	168	}
	169	n := min(len(syms), w.ChunkRows)
	170	if err := w.St.AddSymbols(x.ID, syms[:n]); err != nil {
	171	return false, err
	172	}
	173	syms = syms[n:]
	174	if w.chunkHook != nil {
	175	w.chunkHook(x.ID)
	176	}
	177	}
	178	if err := w.St.PublishSymbolIndex(x); err != nil {
	179	return false, err
	180	}
	181	if err := w.St.PurgeSymbolIndexes(repo.ID); err != nil {
	182	return false, err
122	183	}
123	184	if x.State != "ok" {
124		return fmt.Errorf("%s %s: %s", repo.Path(), x.State, x.Note)
	185	return false, fmt.Errorf("%s %s: %s", repo.Path(), x.State, x.Note)
125	186	}
126		return nil
	187	return false, nil
127	188	}
128	189
129		var errSymbolCap = errors.New("symbol cap reached")
	190	var (
	191	errSymbolCap = errors.New("symbol cap reached")
	192	errByteBudget = errors.New("byte budget reached")
	193	)
130	194
131	195	// collect reads every indexable blob in tree and extracts its symbols,
132		// stopping at the symbol cap or the time bound with what it has.
	196	// stopping at the symbol cap, the byte budget or the time bound with what
	197	// it has.
133	198	func (w *Worker) collect(ctx context.Context, dir, tree string) ([]store.SymbolRow, int, error) {
134	199	ctx, cancel := context.WithTimeout(ctx, w.MaxTime)
135	200	defer cancel()
		@@ -149,8 +214,8 @@ func (w *Worker) collect(ctx context.Context, dir, tree string) ([]store.SymbolR
149	214	shas = append(shas, b.SHA)
150	215	}
151	216	var out []store.SymbolRow
152		files := 0
153		capped := false
	217	files, used := 0, 0
	218	var stop error
154	219	err = gitutil.CatBlobs(ctx, dir, shas, func(i int, data []byte) bool {
155	220	if gitutil.IsBinary(data) {
156	221	return true
		@@ -158,15 +223,22 @@ func (w *Worker) collect(ctx context.Context, dir, tree string) ([]store.SymbolR
158	223	files++
159	224	for _, s := range Extract(paths[i], data) {
160	225	if len(out) >= w.MaxSymbols {
161		capped = true
	226	stop = errSymbolCap
	227	return false
	228	}
	229	// The key is counted too: it is stored beside the name.
	230	n := len(s.Name) + len(s.Key) + len(paths[i])
	231	if used+n > w.MaxBytes {
	232	stop = errByteBudget
162	233	return false
163	234	}
	235	used += n
164	236	out = append(out, store.SymbolRow{Name: s.Name, Key: s.Key, Kind: s.Kind, Path: paths[i], Line: s.Line})
165	237	}
166	238	return true
167	239	})
168		if capped {
169		return out, files, errSymbolCap
	240	if stop != nil {
	241	return out, files, stop
170	242	}
171	243	if err != nil && ctx.Err() == nil {
172	244	return nil, files, err

internal/symbols/worker_test.go +168 −12

		@@ -2,6 +2,7 @@ package symbols
2	2
3	3	import (
4	4	"context"
	5	"fmt"
5	6	"os"
6	7	"os/exec"
7	8	"path/filepath"
		@@ -45,8 +46,7 @@ func newFixture(t testing.T) fixture {
45	46	}
46	47	root := t.TempDir()
47	48	f := &fixture{t: t, st: st, repo: repo, src: filepath.Join(root, "src"), bare: filepath.Join(root, "app.git")}
48		f.w = &Worker{St: st, RepoDir: func(owner, name string) string { return f.bare },
49		MaxSymbols: DefaultMaxSymbols, MaxTime: DefaultMaxTime}
	49	f.w = NewWith(st, func(owner, name string) string { return f.bare }, 0)
50	50	f.git(root, "init", "-q", "-b", repo.DefaultBranch, "src")
51	51	f.git(root, "init", "-q", "--bare", f.bare)
52	52	return f
		@@ -202,22 +202,178 @@ func TestWorkerTimeBound(t *testing.T) {
202	202	}
203	203	}
204	204
205		// A tree that cannot be read is recorded as failed, and the record stands
206		// for that tree: asking again does not retry it.
207		func TestWorkerFailureIsRecordedNotRetried(t *testing.T) {
	205	// breakBlob deletes the loose object of path at the default branch, so
	206	// the tree cannot be read.
	207	func (f *fixture) breakBlob(path string) {
	208	f.t.Helper()
	209	blob := f.git(f.bare, "rev-parse", "refs/heads/"+f.repo.DefaultBranch+":"+path)
	210	if err := os.Remove(filepath.Join(f.bare, "objects", blob[:2], blob[2:])); err != nil {
	211	f.t.Fatal(err)
	212	}
	213	}
	214
	215	// A tree that cannot be read is recorded as a failure beside the current
	216	// index, which stays current. The request is kept for one retry after the
	217	// backoff rather than tried again at once; a new request for the same
	218	// tree inside the backoff is not retried either.
	219	func TestWorkerFailureKeepsCurrentIndex(t *testing.T) {
208	220	f := newFixture(t)
209	221	f.commit(map[string]string{"a.go": "package a\n\nfunc A() {}\n"})
210		blob := f.git(f.bare, "rev-parse", "refs/heads/"+f.repo.DefaultBranch+":a.go")
211		if err := os.Remove(filepath.Join(f.bare, "objects", blob[:2], blob[2:])); err != nil {
	222	good := f.sweep(false)
	223
	224	f.commit(map[string]string{"b.go": "package a\n\nfunc B() {}\n"})
	225	f.breakBlob("b.go")
	226	f.st.RequestSymbolIndex(f.repo.ID, false)
	227	f.w.Sweep(context.Background())
	228	cur, err := f.st.SymbolIndexFor(f.repo.ID)
	229	if err != nil \|\| cur.ID != good.ID {
	230	t.Fatalf("current index after a failure = %+v, %v; want %d", cur, err, good.ID)
	231	}
	232	fail, err := f.st.SymbolFailureFor(f.repo.ID)
	233	if err != nil \|\| !strings.Contains(fail.Note, "b.go") {
	234	t.Fatalf("failure = %+v, %v", fail, err)
	235	}
	236	if reqs, _ := f.st.SymbolRequests(); len(reqs) != 0 {
	237	t.Fatalf("a failed request is due again at once: %+v", reqs)
	238	}
	239	var attempts int
	240	f.st.DB.QueryRow("SELECT attempts FROM symbol_requests WHERE repo_id = ?", f.repo.ID).Scan(&attempts)
	241	if attempts != 1 {
	242	t.Fatalf("attempts = %d, want the request kept for one retry", attempts)
	243	}
	244
	245	// Asked again inside the backoff: the failed tree is left alone.
	246	f.st.RequestSymbolIndex(f.repo.ID, false)
	247	failed, err := f.w.Index(context.Background(), f.repo.ID, false)
	248	if failed \|\| err != nil {
	249	t.Fatalf("a recently failed tree was retried: %v, %v", failed, err)
	250	}
	251
	252	// After the backoff it is tried again, and still fails.
	253	f.w.Backoff = 0
	254	if failed, _ := f.w.Index(context.Background(), f.repo.ID, false); !failed {
	255	t.Fatal("a failed tree was not retried after the backoff")
	256	}
	257
	258	// A second failure of a retry is the end of it.
	259	f.st.DB.Exec("DELETE FROM symbol_requests")
	260	f.st.DB.Exec("INSERT INTO symbol_requests (repo_id, attempts) VALUES (?, 1)", f.repo.ID)
	261	f.w.Sweep(context.Background())
	262	var n int
	263	f.st.DB.QueryRow("SELECT COUNT(*) FROM symbol_requests").Scan(&n)
	264	if n != 0 {
	265	t.Fatal("a failed retry was kept for another")
	266	}
	267
	268	// A push that changes the tree builds, and clears the failure.
	269	f.git(f.src, "rm", "-q", "b.go")
	270	f.commit(map[string]string{"c.go": "package a\n\nfunc C() {}\n"})
	271	next := f.sweep(false)
	272	if next.ID == good.ID {
	273	t.Fatal("a new tree after a failure was not indexed")
	274	}
	275	if _, err := f.st.SymbolFailureFor(f.repo.ID); err != store.ErrNotFound {
	276	t.Fatalf("failure kept after a good index: %v", err)
	277	}
	278	}
	279
	280	// A new index is written in chunks no read sees: throughout the build the
	281	// old index is current, and the flip replaces it whole.
	282	func TestWorkerBuildIsInvisibleUntilPublished(t *testing.T) {
	283	f := newFixture(t)
	284	f.commit(map[string]string{"a.go": "package a\n\nfunc Old() {}\n"})
	285	old := f.sweep(false)
	286
	287	f.commit(map[string]string{"a.go": "package a\n\nfunc New1() {}\nfunc New2() {}\nfunc New3() {}\n"})
	288	f.w.ChunkRows = 1
	289	chunks := 0
	290	f.w.chunkHook = func(building int64) {
	291	chunks++
	292	cur, err := f.st.SymbolIndexFor(f.repo.ID)
	293	if err != nil \|\| cur.ID != old.ID {
	294	t.Errorf("mid-build current index = %+v, %v; want the old one", cur, err)
	295	}
	296	if got := f.names(cur); len(got) != 1 \|\| got[0] != "a.go:Old" {
	297	t.Errorf("mid-build reads %v", got)
	298	}
	299	if b, _ := f.st.SymbolIndexByID(building); b.State != "building" {
	300	t.Errorf("index being built is %q", b.State)
	301	}
	302	}
	303	next := f.sweep(false)
	304	if chunks != 3 {
	305	t.Fatalf("%d chunks, want one per row", chunks)
	306	}
	307	if got := f.names(next); len(got) != 3 {
	308	t.Fatalf("after the flip: %v", got)
	309	}
	310	var indexes, rows int
	311	f.st.DB.QueryRow("SELECT COUNT(*) FROM symbol_indexes").Scan(&indexes)
	312	f.st.DB.QueryRow("SELECT COUNT(*) FROM symbols").Scan(&rows)
	313	if indexes != 1 \|\| rows != 3 {
	314	t.Fatalf("%d indexes and %d rows left, want the new index alone", indexes, rows)
	315	}
	316	}
	317
	318	// A run that stopped mid-build leaves the old index current, and the next
	319	// run deletes what it wrote.
	320	func TestWorkerCleansUpInterruptedBuild(t *testing.T) {
	321	f := newFixture(t)
	322	f.commit(map[string]string{"a.go": "package a\n\nfunc Old() {}\n"})
	323	old := f.sweep(false)
	324
	325	// What a crash between chunks leaves.
	326	orphan, err := f.st.BeginSymbolIndex(f.repo.ID, "c", "t")
	327	if err != nil {
	328	t.Fatal(err)
	329	}
	330	if err := f.st.AddSymbols(orphan, []store.SymbolRow{{Name: "Half", Key: "Half", Kind: "function", Path: "x.go", Line: 1}}); err != nil {
212	331	t.Fatal(err)
213	332	}
	333	if cur, _ := f.st.SymbolIndexFor(f.repo.ID); cur.ID != old.ID {
	334	t.Fatalf("an unfinished build became current: %+v", cur)
	335	}
	336
	337	f.commit(nil) // same tree: nothing to build, but the leftovers go
	338	f.sweep(false)
	339	if _, err := f.st.SymbolIndexByID(orphan); err != store.ErrNotFound {
	340	t.Fatalf("interrupted build still there: %v", err)
	341	}
	342	var rows int
	343	f.st.DB.QueryRow("SELECT COUNT(*) FROM symbols WHERE index_id = ?", orphan).Scan(&rows)
	344	if rows != 0 {
	345	t.Fatalf("%d rows of the interrupted build left", rows)
	346	}
	347	}
	348
	349	// A hostile tree: names past the length cap are dropped, and long
	350	// headings stop at the byte budget with a partial index saying so.
	351	func TestWorkerHostileNames(t *testing.T) {
	352	f := newFixture(t)
	353	files := map[string]string{}
	354	huge := strings.Repeat("x", MaxNameBytes+1)
	355	for i := 0; i < 20; i++ {
	356	files[fmt.Sprintf("huge%02d.go", i)] = "package a\n\nfunc " + huge + "() {}\nvar " + huge + " int\n"
	357	}
	358	var md strings.Builder
	359	for i := 0; i < 500; i++ {
	360	fmt.Fprintf(&md, "# %03d %s\n", i, strings.Repeat("h", 200))
	361	}
	362	files["notes.md"] = md.String()
	363	f.commit(files)
	364	f.w.MaxBytes = 20_000
214	365	x := f.sweep(false)
215		if x.State != "failed" \|\| x.Note == "" \|\| x.Symbols != 0 {
216		t.Fatalf("index = %+v, want failed with a note", x)
	366	if x.State != "partial" \|\| !strings.Contains(x.Note, "20000 bytes") {
	367	t.Fatalf("index = %+v, want partial at the byte budget", x)
	368	}
	369	var longest, total int
	370	f.st.DB.QueryRow("SELECT COALESCE(MAX(length(name)), 0), COALESCE(SUM(length(name) + length(key) + length(path)), 0) FROM symbols").
	371	Scan(&longest, &total)
	372	if longest > MaxNameBytes {
	373	t.Errorf("a %d-byte name was kept", longest)
217	374	}
218		again := f.sweep(false)
219		if again.ID != x.ID {
220		t.Fatalf("a failed tree was retried: %+v then %+v", x, again)
	375	if total > 20_000 \|\| x.Symbols == 0 {
	376	t.Errorf("%d symbols holding %d bytes, want some within the budget", x.Symbols, total)
221	377	}
222	378	}
223	379

Commit c528009533

Commit `c528009533`