Commit `c528009533`

c52800953399a64c7a80d7970a8772dfe17873f8

parent: 7a781d1ccb

Verified · cmc ci/build: success ci/test: success

cmc <hello@cleberg.net> · 2026-09-29 05:56 UTC

symbols: chunked builds, failure records, name and byte bounds

A new index is written in 5000-row transactions under a building id no
read sees, published in one short transaction, and the replaced one is
deleted in chunks; the next run removes an interrupted build. A failed
run is recorded beside the current index, which stays in use, and is
retried once after an hour or on the next push. Names over 256 bytes
are dropped and a run stops at 32 MiB of names and paths. The blob
view resolves its ref once and passes the commit to IndexedTree; repo
symbols and the results page take queries of two characters or more,
and the page's cursor names its index.

Closes #293

Layout: unified · split

.gitbay/wiki/Users.org +14 −8

	@@ -252,7 +252,8 @@ gitbay repo unarchive you/project # writes refused, browsing
252	A push to the default branch queues a symbol index of its tree, built in	252	A push to the default branch queues a symbol index of its tree, built in
253	the background after the push completes; a tree that is already indexed	253	the background after the push completes; a tree that is already indexed
254	is not indexed again. =repo symbols <owner/name> [--kind k] <query>=	254	is not indexed again. =repo symbols <owner/name> [--kind k] <query>=
255	lists definitions whose name starts with the query, exact matches	255	lists definitions whose name starts with the query (two characters
		256	at least), exact matches
256	first, case-sensitive before case-insensitive, as name, kind, path and	257	first, case-sensitive before case-insensitive, as name, kind, path and
257	line; =--limit= and =--cursor= page it, and without them it stops at	258	line; =--limit= and =--cursor= page it, and without them it stops at
258	200 rows. A Go method is listed as =Type.Method= and also matches on	259	200 rows. A Go method is listed as =Type.Method= and also matches on
	@@ -276,13 +277,18 @@ What is indexed:
276	Languages other than Go are matched a line at a time, so unusual	277	Languages other than Go are matched a line at a time, so unusual
277	definition shapes are missed. Files over 1 MiB, anything under	278	definition shapes are missed. Files over 1 MiB, anything under
278	=vendor/= or =node_modules/=, =_gen.go=, =.pb.go=, Go files marked	279	=vendor/= or =node_modules/=, =_gen.go=, =.pb.go=, Go files marked
279	=Code generated ... DO NOT EDIT.=, and =*.min.js= are skipped. One run	280	=Code generated ... DO NOT EDIT.=, and =*.min.js= are skipped, and a
280	stops at 200,000 symbols or two minutes and records the index as	281	name longer than 256 bytes is dropped. One run stops at 200,000
281	partial; an index that cannot be built is recorded as failed. Either	282	symbols, 32 MiB of names and paths, or two minutes, and publishes what
282	record stands for its tree: a push that changes the tree builds again,	283	it found as a partial index that says which bound it reached.
283	one that does not is left alone. A repository that has not been pushed	284
284	to since the index existed has none until its next push to the default	285	A new index is written while the previous one stays in use, and
285	branch. =admin symbols reindex <owner/name>= rebuilds one regardless.	286	replaces it in one step. A run that cannot build an index records why
		287	and leaves the previous index current; the same tree is tried again
		288	once after an hour, or when a push changes the tree. A repository that
		289	has not been pushed to since the index existed has none until its next
		290	push to the default branch. =admin symbols reindex <owner/name>=
		291	rebuilds one regardless.
286		292
287	On the web, a file viewed at the indexed tree (the default branch's	293	On the web, a file viewed at the indexed tree (the default branch's
288	head, or any commit with the same tree) links each name the index	294	head, or any commit with the same tree) links each name the index

internal/control/symbols.go +31 −20

 	if path == "" || query == "" {
 		return c.usage()
 	}
-	if len(query) > maxQueryLen {
+	if len(query) < MinSymbolQuery || len(query) > maxQueryLen {
-		return c.fail(protocol.ExitUsage, "query must be 1 to %d characters", maxQueryLen)
+		return c.fail(protocol.ExitUsage, "query must be %d to %d characters", MinSymbolQuery, maxQueryLen)
 	}
 	if kind != "" && !symbols.ValidKind(kind) {
 		return c.fail(protocol.ExitUsage, "--kind must be one of %s", strings.Join(symbols.Kinds, ", "))
 	}
 	idx, err := c.Store.SymbolIndexFor(repo.ID)
 	if errors.Is(err, store.ErrNotFound) {
+		if f, ferr := c.Store.SymbolFailureFor(repo.ID); ferr == nil {
+			return c.fail(protocol.ExitFailure, "the symbol index of %s could not be built: %s", repo.Path(), f.Note)
+		}
 		return c.fail(protocol.ExitNotFound, "%s has no symbol index yet; one is built after a push to %s", repo.Path(), repo.DefaultBranch)
 	} else if err != nil {
 		return c.fail(protocol.ExitFailure, "%v", err)
 	}
 	if ref != "" && ref != repo.DefaultBranch {
 		dir := RepoDir(c.Cfg.Server.Root, repo.OwnerName, repo.Name)
-		if !IndexedTree(dir, ref, idx) {
+		sha, err := gitutil.ResolveRef(dir, ref)
+		if err != nil || !IndexedTree(dir, sha, idx) {
 			return c.fail(protocol.ExitNotFound, "only the default branch, %s, is indexed; %s is not at the indexed tree", repo.DefaultBranch, ref)
 		}
 	}
-	if idx.State == "failed" {
-		return c.fail(protocol.ExitFailure, "the symbol index of %s failed: %s", repo.Path(), idx.Note)
-	}
 	var after int64
 	if p.key != "" {
-		cursorIdx, id, ok := parseSymbolCursor(p.key)
+		cursorIdx, id, ok := ParseSymbolCursor(p.key)
 		if !ok {
 			return c.fail(protocol.ExitUsage, "bad cursor")
 		}
 		if cursorIdx != idx.ID {
-			return c.fail(protocol.ExitUsage, "the symbol index was rebuilt since that cursor; start again without --cursor")
+			return c.fail(protocol.ExitUsage, "%s", StaleSymbolCursor)
 		}
 		after = id
 	}
 		rows = rows[:symbolsUnpaged]
 	}
 	rows, next := trimPage(p, rows, "symbol", func(r store.SymbolRow) string {
-		return strconv.FormatInt(idx.ID, 10) + "." + strconv.FormatInt(r.ID, 10)
+		return SymbolCursor(idx.ID, r.ID)
 	})
 	var ds []symbolOut
 	for _, r := range rows {
 	})
 }
-// parseSymbolCursor reads "<index id>.<row id>". The index id makes a
+// MinSymbolQuery is the shortest query repo symbols and the results page
+// take: one character matches too much of an index to be worth a page.
+const MinSymbolQuery = 2
+// StaleSymbolCursor is the refusal for a cursor taken from an index that
+// has since been replaced.
+const StaleSymbolCursor = "the symbol index was rebuilt since that cursor; start again without it"
+// SymbolCursor is the paging key after row id of index indexID, the form
+// ParseSymbolCursor reads.
+func SymbolCursor(indexID, id int64) string {
+	return strconv.FormatInt(indexID, 10) + "." + strconv.FormatInt(id, 10)
+}
+// ParseSymbolCursor reads "<index id>.<row id>". The index id makes a
 // cursor from before a rebuild fail rather than page through the new
 // index from an unrelated row.
-func parseSymbolCursor(key string) (int64, int64, bool) {
+func ParseSymbolCursor(key string) (int64, int64, bool) {
 	a, b, ok := strings.Cut(key, ".")
 	if !ok {
 		return 0, 0, false
 	return idx, id, err1 == nil && err2 == nil && id > 0
 }
-// IndexedTree reports whether ref names a commit whose tree is the one
+// IndexedTree reports whether commit sha has the tree idx was built from:
-// idx was built from: the default branch's head when the index is
+// the default branch's head when the index is current, or any other
-// current, or any other commit with the same content. The blob view uses
+// commit with the same content. The blob view uses it to decide whether
-// it to decide whether its names can link into the index.
+// its names can link into the index.
-func IndexedTree(dir, ref string, idx store.SymbolIndex) bool {
+func IndexedTree(dir, sha string, idx store.SymbolIndex) bool {
-	sha, err := gitutil.ResolveRef(dir, ref)
-	if err != nil {
-		return false
-	}
 	if sha == idx.Commit {
 		return true
 	}

internal/control/symbols_test.go +1

	@@ -114,6 +114,7 @@ func TestRepoSymbolsRefusals(t *testing.T) {
114	{"another ref", alice, []string{"repo", "symbols", repo.Path(), "--ref", "feature", "Parse"}, protocol.ExitNotFound, "only the default branch"},	114	{"another ref", alice, []string{"repo", "symbols", repo.Path(), "--ref", "feature", "Parse"}, protocol.ExitNotFound, "only the default branch"},
115	{"unknown kind", alice, []string{"repo", "symbols", repo.Path(), "--kind", "widget", "Parse"}, protocol.ExitUsage, "--kind"},	115	{"unknown kind", alice, []string{"repo", "symbols", repo.Path(), "--kind", "widget", "Parse"}, protocol.ExitUsage, "--kind"},
116	{"no query", alice, []string{"repo", "symbols", repo.Path()}, protocol.ExitUsage, ""},	116	{"no query", alice, []string{"repo", "symbols", repo.Path()}, protocol.ExitUsage, ""},
		117	{"one-character query", alice, []string{"repo", "symbols", repo.Path(), "P"}, protocol.ExitUsage, "2 to"},
117	{"reindex needs an admin", alice, []string{"admin", "symbols", "reindex", repo.Path()}, protocol.ExitDenied, "admin"},	118	{"reindex needs an admin", alice, []string{"admin", "symbols", "reindex", repo.Path()}, protocol.ExitDenied, "admin"},
118	} {	119	} {
119	t.Run(tc.name, func(t *testing.T) {	120	t.Run(tc.name, func(t *testing.T) {

internal/httpd/symbols.go +25 −12

 import (
 	"errors"
+	"fmt"
 	"html/template"
 	"net/http"
 	"net/url"
 }
 // blobSymbols finds the viewed file's symbols and links the names in its
-// highlighted source to their definitions, when the ref being viewed has
+// highlighted source to their definitions, when commit, the ref being
-// the indexed tree. A name defined once links to that line at the same
+// viewed as the page resolved it, has the indexed tree. A name defined once links to that line at the same
 // ref; one defined more than once links to the results page. Names the
 // index does not hold stay plain. The whole page costs one lookup of the
 // file's distinct names, however many times each appears.
-func (s *Server) blobSymbols(p repoPage, filePath string, code template.HTML) (template.HTML, []store.SymbolRow) {
+func (s *Server) blobSymbols(p repoPage, commit, filePath string, code template.HTML) (template.HTML, []store.SymbolRow) {
 	idx, err := s.st.SymbolIndexFor(p.Repo.ID)
-	if err != nil || idx.State == "failed" || !control.IndexedTree(p.Dir, p.Ref, idx) {
+	if err != nil || !control.IndexedTree(p.Dir, commit, idx) {
 		return code, nil
 	}
 	list, _ := s.st.SymbolsInFile(idx.ID, filePath)
 	switch {
 	case errors.Is(err, store.ErrNotFound):
 		problem = "no symbol index yet; one is built after a push to " + p.Repo.DefaultBranch
+		if f, ferr := s.st.SymbolFailureFor(p.Repo.ID); ferr == nil {
+			problem = "the symbol index could not be built: " + f.Note
+		}
 	case err != nil:
 		http.Error(w, "internal error", http.StatusInternalServerError)
 		return
-	case idx.State == "failed":
+	case q == "":
-		problem = "the symbol index failed: " + idx.Note
+	case len(q) < control.MinSymbolQuery || len(q) > 200:
-	case len(q) > 200:
+		problem = fmt.Sprintf("query must be %d to 200 characters", control.MinSymbolQuery)
-		problem = "query must be 1 to 200 characters"
+	default:
-	case q != "":
 		if idx.State == "partial" {
 			note = "the index is partial: " + idx.Note
 		}
 		var after int64
-		if a, err := strconv.ParseInt(r.URL.Query().Get("after"), 10, 64); err == nil && a > 0 {
+		if a := r.URL.Query().Get("after"); a != "" {
-			after = a
+			cursorIdx, id, ok := control.ParseSymbolCursor(a)
+			switch {
+			case !ok:
+				problem = "bad cursor"
+			case cursorIdx != idx.ID:
+				problem = control.StaleSymbolCursor
+			}
+			after = id
+		}
+		if problem != "" {
+			break
 		}
 		rows, err = s.st.SearchSymbols(idx.ID, q, kind, symbolsPageSize+1, after)
 		if err != nil {
 		}
 		if len(rows) > symbolsPageSize {
 			rows = rows[:symbolsPageSize]
-			v := url.Values{"q": {q}, "after": {strconv.FormatInt(rows[len(rows)-1].ID, 10)}}
+			v := url.Values{"q": {q}, "after": {control.SymbolCursor(idx.ID, rows[len(rows)-1].ID)}}
 			if kind != "" {
 				v.Set("kind", kind)
 			}

internal/httpd/symbols_test.go +10 −3

 	symbolGit(t, src, "add", ".")
 	symbolGit(t, src, "commit", "-q", "-m", "side")
-	w := &symbols.Worker{St: st, RepoDir: func(owner, name string) string { return control.RepoDir(root, owner, name) },
+	w := symbols.NewWith(st, func(owner, name string) string { return control.RepoDir(root, owner, name) }, 0)
-		MaxSymbols: symbols.DefaultMaxSymbols, MaxTime: symbols.DefaultMaxTime}
 	for _, r := range []struct{ name, vis string }{{"app", "public"}, {"secret", "private"}} {
 		id, err := st.CreateRepo("user", uid, r.name, r.vis)
 		if err != nil {
 		os.MkdirAll(filepath.Dir(dir), 0o755)
 		symbolGit(t, root, "init", "-q", "--bare", dir)
 		symbolGit(t, src, "push", "-q", dir, "main", "side")
-		if err := w.Index(context.Background(), id, false); err != nil {
+		if _, err := w.Index(context.Background(), id, false); err != nil {
 			t.Fatal(err)
 		}
 	}
 	if strings.Contains(w.Body.String(), "a/dup.go:3") {
 		t.Error("kind filter ignored")
 	}
+	if w := get(t, h, "/alice/app/symbols?q=d", nil); !strings.Contains(w.Body.String(), "query must be 2 to 200 characters") {
+		t.Error("a one-character query was searched")
+	}
+	// A cursor from an index that has been replaced says so rather than
+	// showing an empty page.
+	if w := get(t, h, "/alice/app/symbols?q=dup&after=999999.1", nil); !strings.Contains(w.Body.String(), "rebuilt since that cursor") {
+		t.Errorf("stale cursor:\n%s", w.Body.String())
+	}
 	for _, p := range []string{"/alice/secret/symbols?q=dup", "/alice/secret/blob/main/main.go"} {
 		if w := get(t, h, p, nil); w.Code != 404 {
 			t.Errorf("%s anonymously: %d, want 404", p, w.Code)

internal/httpd/web.go +9 −2

 	}
 	p.Tab = "files"
 	filePath := strings.Trim(r.PathValue("path"), "/")
-	data, err := gitutil.ReadBlob(p.Dir, p.Ref, filePath, maxRenderBytes+1)
+	// The file and its symbol links are read from one commit, even if the
+	// ref moves while the page renders.
+	commit, err := gitutil.ResolveRef(p.Dir, p.Ref)
+	if err != nil {
+		s.notFound(w, r)
+		return
+	}
+	data, err := gitutil.ReadBlob(p.Dir, commit, filePath, maxRenderBytes+1)
 	if err != nil {
 		s.notFound(w, r)
 		return
 	}
 	var fileSymbols []store.SymbolRow
 	if !binary && !image {
-		codeHTML, fileSymbols = s.blobSymbols(p, filePath, codeHTML)
+		codeHTML, fileSymbols = s.blobSymbols(p, commit, filePath, codeHTML)
 	}
 	// Markdown and org render like a README, with the source one click
 	// away; ?view=source shows the text instead.

internal/store/migrations/0072_symbols.down.sql +1

	@@ -1,3 +1,4 @@
1	DROP TABLE symbol_requests;	1	DROP TABLE symbol_requests;
		2	DROP TABLE symbol_failures;
2	DROP TABLE symbols;	3	DROP TABLE symbols;
3	DROP TABLE symbol_indexes;	4	DROP TABLE symbol_indexes;

internal/store/migrations/0072_symbols.up.sql +27 −2

+-- Symbol indexes (#293): the definitions found in the tree of a
+-- repository's default branch head. tree is the key: a head whose tree
+-- is the current index's needs no new one.
+--
+-- An index is written in chunks while its state is building, which no
+-- read sees; publishing it makes it ok or partial (a bound was reached,
+-- note says which) and retires the previous one in one short
+-- transaction. Building and retired rows are deleted, symbols first and
+-- in chunks, by the worker. At most one index per repository is current.
+--
 -- AUTOINCREMENT because a replaced index must not hand its id to the
 -- next: a paging cursor names the index it was taken from.
 CREATE TABLE symbol_indexes (
     id         INTEGER PRIMARY KEY AUTOINCREMENT,
-    repo_id    INTEGER NOT NULL UNIQUE REFERENCES repos(id) ON DELETE CASCADE,
+    repo_id    INTEGER NOT NULL REFERENCES repos(id) ON DELETE CASCADE,
     commit_sha TEXT NOT NULL,
     tree       TEXT NOT NULL,
-    state      TEXT NOT NULL CHECK (state IN ('ok', 'partial', 'failed')),
+    state      TEXT NOT NULL CHECK (state IN ('building', 'ok', 'partial', 'retired')),
     note       TEXT NOT NULL DEFAULT '',
     files      INTEGER NOT NULL DEFAULT 0,
     symbols    INTEGER NOT NULL DEFAULT 0,
     built_at   TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
 );
+CREATE INDEX symbol_indexes_repo ON symbol_indexes(repo_id, state);
+CREATE UNIQUE INDEX symbol_indexes_current ON symbol_indexes(repo_id)
+    WHERE state IN ('ok', 'partial');
 -- name is what is listed, key the name as written at a use site (they
 -- differ for Go methods: Type.Method and Method). lname and lkey are
 -- their lower-case forms, for case-insensitive prefix ranges.
 CREATE INDEX symbols_lkey ON symbols(index_id, lkey);
 CREATE INDEX symbols_key ON symbols(index_id, key);
 CREATE INDEX symbols_path ON symbols(index_id, path, line);
+-- The last run that could not build an index, kept apart so the current
+-- index stays in place. Cleared by the next index published.
+CREATE TABLE symbol_failures (
+    repo_id   INTEGER PRIMARY KEY REFERENCES repos(id) ON DELETE CASCADE,
+    tree      TEXT NOT NULL,
+    note      TEXT NOT NULL,
+    failed_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
+);
 -- Repositories waiting for the index worker. seq counts requests, so a
 -- push that lands while a build runs leaves its request in place; force
+-- rebuilds even when the tree is the indexed one. not_before defers a
+-- retry after a failure; attempts counts those retries.
 CREATE TABLE symbol_requests (
     repo_id      INTEGER PRIMARY KEY REFERENCES repos(id) ON DELETE CASCADE,
     seq          INTEGER NOT NULL DEFAULT 1,
     force        INTEGER NOT NULL DEFAULT 0,
+    attempts     INTEGER NOT NULL DEFAULT 0,
+    not_before   TEXT NOT NULL DEFAULT '',
     requested_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
 );

internal/store/symbols.go +199 −38

	@@ -3,24 +3,32 @@ package store
3	import (	3	import (
4	"database/sql"	4	"database/sql"
5	"errors"	5	"errors"
		6	"fmt"
6	"strings"	7	"strings"
7	"unicode/utf8"	8	"unicode/utf8"
8	)	9	)
9		10
10	// SymbolIndex is a repository's symbol index: what was indexed and how it	11	// SymbolIndex is one symbol index of a repository. The states are
11	// went. The states are described with the table's schema.	12	// described with the table's schema; reads only ever see ok or partial.
12	type SymbolIndex struct {	13	type SymbolIndex struct {
13	ID int64	14	ID int64
14	RepoID int64	15	RepoID int64
15	Commit string	16	Commit string
16	Tree string	17	Tree string
17	State string // ok \| partial \| failed	18	State string // building \| ok \| partial \| retired
18	Note string	19	Note string
19	Files int	20	Files int
20	Symbols int	21	Symbols int
21	BuiltAt string	22	BuiltAt string
22	}	23	}
23		24
		25	// SymbolFailure is the last run that could not build an index.
		26	type SymbolFailure struct {
		27	Tree string
		28	Note string
		29	FailedAt string
		30	}
		31
24	// SymbolRow is one definition. Key is the name as written where it is	32	// SymbolRow is one definition. Key is the name as written where it is
25	// used; see internal/symbols.	33	// used; see internal/symbols.
26	type SymbolRow struct {	34	type SymbolRow struct {
	@@ -42,26 +50,31 @@ type SymbolTarget struct {
42		50
43	// SymbolRequest is a repository waiting for the index worker.	51	// SymbolRequest is a repository waiting for the index worker.
44	type SymbolRequest struct {	52	type SymbolRequest struct {
45	RepoID int64	53	RepoID int64
46	Seq int64	54	Seq int64
47	Force bool	55	Force bool
		56	Attempts int
48	}	57	}
49		58
50	// RequestSymbolIndex queues a repository for the index worker. A request	59	// RequestSymbolIndex queues a repository for the index worker. A request
51	// already waiting is bumped, so one taken by a running build is not	60	// already waiting is bumped, so one taken by a running build is not
52	// cleared when that build ends; force is kept once set.	61	// cleared when that build ends, and a deferred retry becomes due now;
		62	// force is kept once set.
53	func (s *Store) RequestSymbolIndex(repoID int64, force bool) error {	63	func (s *Store) RequestSymbolIndex(repoID int64, force bool) error {
54	_, err := s.DB.Exec(`	64	_, err := s.DB.Exec(`
55	INSERT INTO symbol_requests (repo_id, force) VALUES (?, ?)	65	INSERT INTO symbol_requests (repo_id, force) VALUES (?, ?)
56	ON CONFLICT (repo_id) DO UPDATE SET seq = seq + 1,	66	ON CONFLICT (repo_id) DO UPDATE SET seq = seq + 1,
57	force = MAX(force, excluded.force),	67	force = MAX(force, excluded.force), attempts = 0, not_before = '',
58	requested_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')`, repoID, force)	68	requested_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')`, repoID, force)
59	return err	69	return err
60	}	70	}
61		71
62	// SymbolRequests lists the waiting repositories, oldest request first.	72	// SymbolRequests lists the requests that are due, oldest first.
63	func (s *Store) SymbolRequests() ([]SymbolRequest, error) {	73	func (s *Store) SymbolRequests() ([]SymbolRequest, error) {
64	rows, err := s.DB.Query(`SELECT repo_id, seq, force FROM symbol_requests ORDER BY requested_at, repo_id`)	74	rows, err := s.DB.Query(`
		75	SELECT repo_id, seq, force, attempts FROM symbol_requests
		76	WHERE not_before = '' OR not_before <= strftime('%Y-%m-%dT%H:%M:%fZ','now')
		77	ORDER BY requested_at, repo_id`)
65	if err != nil {	78	if err != nil {
66	return nil, err	79	return nil, err
67	}	80	}
	@@ -69,7 +82,7 @@ func (s *Store) SymbolRequests() ([]SymbolRequest, error) {
69	var out []SymbolRequest	82	var out []SymbolRequest
70	for rows.Next() {	83	for rows.Next() {
71	var r SymbolRequest	84	var r SymbolRequest
72	if err := rows.Scan(&r.RepoID, &r.Seq, &r.Force); err != nil {	85	if err := rows.Scan(&r.RepoID, &r.Seq, &r.Force, &r.Attempts); err != nil {
73	return nil, err	86	return nil, err
74	}	87	}
75	out = append(out, r)	88	out = append(out, r)
	@@ -84,46 +97,65 @@ func (s *Store) DoneSymbolRequest(r SymbolRequest) error {
84	return err	97	return err
85	}	98	}
86		99
87	// SymbolIndexFor returns a repository's index, or ErrNotFound when none	100	// DeferSymbolRequest keeps a request for another attempt after seconds,
88	// has been built.	101	// unless it was requested again since it was read.
89	func (s *Store) SymbolIndexFor(repoID int64) (SymbolIndex, error) {	102	func (s *Store) DeferSymbolRequest(r SymbolRequest, seconds int) error {
		103	_, err := s.DB.Exec(`
		104	UPDATE symbol_requests SET force = 0, attempts = attempts + 1,
		105	not_before = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', ?)
		106	WHERE repo_id = ? AND seq = ?`, fmt.Sprintf("+%d seconds", seconds), r.RepoID, r.Seq)
		107	return err
		108	}
		109
		110	const symbolIndexCols = `id, repo_id, commit_sha, tree, state, note, files, symbols, built_at`
		111
		112	func scanSymbolIndex(row *sql.Row) (SymbolIndex, error) {
90	var x SymbolIndex	113	var x SymbolIndex
91	err := s.DB.QueryRow(`	114	err := row.Scan(&x.ID, &x.RepoID, &x.Commit, &x.Tree, &x.State, &x.Note, &x.Files, &x.Symbols, &x.BuiltAt)
92	SELECT id, repo_id, commit_sha, tree, state, note, files, symbols, built_at
93	FROM symbol_indexes WHERE repo_id = ?`, repoID).
94	Scan(&x.ID, &x.RepoID, &x.Commit, &x.Tree, &x.State, &x.Note, &x.Files, &x.Symbols, &x.BuiltAt)
95	if errors.Is(err, sql.ErrNoRows) {	115	if errors.Is(err, sql.ErrNoRows) {
96	return x, ErrNotFound	116	return x, ErrNotFound
97	}	117	}
98	return x, err	118	return x, err
99	}	119	}
100		120
		121	// SymbolIndexFor returns a repository's current index, or ErrNotFound
		122	// when none has been published.
		123	func (s *Store) SymbolIndexFor(repoID int64) (SymbolIndex, error) {
		124	return scanSymbolIndex(s.DB.QueryRow(`SELECT `+symbolIndexCols+`
		125	FROM symbol_indexes WHERE repo_id = ? AND state IN ('ok', 'partial')`, repoID))
		126	}
		127
		128	// SymbolIndexByID returns an index in any state.
		129	func (s *Store) SymbolIndexByID(id int64) (SymbolIndex, error) {
		130	return scanSymbolIndex(s.DB.QueryRow(`SELECT `+symbolIndexCols+`
		131	FROM symbol_indexes WHERE id = ?`, id))
		132	}
		133
		134	// BeginSymbolIndex creates an index in the building state, which no read
		135	// sees until PublishSymbolIndex.
		136	func (s *Store) BeginSymbolIndex(repoID int64, commit, tree string) (int64, error) {
		137	res, err := s.DB.Exec(`
		138	INSERT INTO symbol_indexes (repo_id, commit_sha, tree, state)
		139	VALUES (?, ?, ?, 'building')`, repoID, commit, tree)
		140	if err != nil {
		141	return 0, err
		142	}
		143	return res.LastInsertId()
		144	}
		145
101	// symbolInsertRows is how many rows one INSERT carries: eight columns	146	// symbolInsertRows is how many rows one INSERT carries: eight columns
102	// each, well under SQLite's variable limit.	147	// each, well under SQLite's variable limit.
103	const symbolInsertRows = 200	148	const symbolInsertRows = 200
104		149
105	// ReplaceSymbolIndex swaps a repository's index for a new one in one	150	// AddSymbols writes rows into an index being built, in one transaction.
106	// transaction: readers see the old index or the new, never a mix. The	151	// The caller keeps each call to a few thousand rows so the write lock is
107	// symbols count is taken from syms.	152	// held briefly.
108	func (s *Store) ReplaceSymbolIndex(x SymbolIndex, syms []SymbolRow) (int64, error) {	153	func (s *Store) AddSymbols(indexID int64, syms []SymbolRow) error {
109	tx, err := s.DB.Begin()	154	tx, err := s.DB.Begin()
110	if err != nil {	155	if err != nil {
111	return 0, err	156	return err
112	}	157	}
113	defer tx.Rollback()	158	defer tx.Rollback()
114	if _, err := tx.Exec("DELETE FROM symbol_indexes WHERE repo_id = ?", x.RepoID); err != nil {
115	return 0, err
116	}
117	res, err := tx.Exec(`
118	INSERT INTO symbol_indexes (repo_id, commit_sha, tree, state, note, files, symbols)
119	VALUES (?, ?, ?, ?, ?, ?, ?)`, x.RepoID, x.Commit, x.Tree, x.State, x.Note, x.Files, len(syms))
120	if err != nil {
121	return 0, err
122	}
123	id, err := res.LastInsertId()
124	if err != nil {
125	return 0, err
126	}
127	for len(syms) > 0 {	159	for len(syms) > 0 {
128	n := min(len(syms), symbolInsertRows)	160	n := min(len(syms), symbolInsertRows)
129	var q strings.Builder	161	var q strings.Builder
	@@ -134,14 +166,143 @@ func (s *Store) ReplaceSymbolIndex(x SymbolIndex, syms []SymbolRow) (int64, erro
134	q.WriteString(",")	166	q.WriteString(",")
135	}	167	}
136	q.WriteString("(?,?,?,?,?,?,?,?)")	168	q.WriteString("(?,?,?,?,?,?,?,?)")
137	args = append(args, id, r.Name, strings.ToLower(r.Name), r.Key, strings.ToLower(r.Key), r.Kind, r.Path, r.Line)	169	args = append(args, indexID, r.Name, strings.ToLower(r.Name), r.Key, strings.ToLower(r.Key), r.Kind, r.Path, r.Line)
138	}	170	}
139	if _, err := tx.Exec(q.String(), args...); err != nil {	171	if _, err := tx.Exec(q.String(), args...); err != nil {
		172	return err
		173	}
		174	syms = syms[n:]
		175	}
		176	return tx.Commit()
		177	}
		178
		179	// PublishSymbolIndex makes a built index the repository's current one, as
		180	// ok or partial, and retires the one it replaces, in one short
		181	// transaction. A recorded failure is cleared.
		182	func (s *Store) PublishSymbolIndex(x SymbolIndex) error {
		183	tx, err := s.DB.Begin()
		184	if err != nil {
		185	return err
		186	}
		187	defer tx.Rollback()
		188	if _, err := tx.Exec(`
		189	UPDATE symbol_indexes SET state = 'retired'
		190	WHERE repo_id = ? AND state IN ('ok', 'partial')`, x.RepoID); err != nil {
		191	return err
		192	}
		193	res, err := tx.Exec(`
		194	UPDATE symbol_indexes SET state = ?, note = ?, files = ?,
		195	symbols = (SELECT COUNT(*) FROM symbols WHERE index_id = ?),
		196	built_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')
		197	WHERE id = ? AND repo_id = ? AND state = 'building'`,
		198	x.State, x.Note, x.Files, x.ID, x.ID, x.RepoID)
		199	if err != nil {
		200	return err
		201	}
		202	if n, _ := res.RowsAffected(); n != 1 {
		203	return fmt.Errorf("symbol index %d is not being built", x.ID)
		204	}
		205	if _, err := tx.Exec("DELETE FROM symbol_failures WHERE repo_id = ?", x.RepoID); err != nil {
		206	return err
		207	}
		208	return tx.Commit()
		209	}
		210
		211	// symbolDeleteRows is how many symbols one purge transaction deletes.
		212	const symbolDeleteRows = 5000
		213
		214	// PurgeSymbolIndexes deletes a repository's indexes that are not current:
		215	// retired ones, and building ones left by a run that did not finish. The
		216	// symbols go a chunk per transaction, then the index row.
		217	func (s *Store) PurgeSymbolIndexes(repoID int64) error {
		218	rows, err := s.DB.Query(`SELECT id FROM symbol_indexes
		219	WHERE repo_id = ? AND state IN ('building', 'retired')`, repoID)
		220	if err != nil {
		221	return err
		222	}
		223	var ids []int64
		224	for rows.Next() {
		225	var id int64
		226	if err := rows.Scan(&id); err != nil {
		227	rows.Close()
		228	return err
		229	}
		230	ids = append(ids, id)
		231	}
		232	rows.Close()
		233	if err := rows.Err(); err != nil {
		234	return err
		235	}
		236	for _, id := range ids {
		237	for {
		238	res, err := s.DB.Exec(`DELETE FROM symbols WHERE id IN
		239	(SELECT id FROM symbols WHERE index_id = ? LIMIT ?)`, id, symbolDeleteRows)
		240	if err != nil {
		241	return err
		242	}
		243	if n, _ := res.RowsAffected(); n == 0 {
		244	break
		245	}
		246	}
		247	if _, err := s.DB.Exec(`DELETE FROM symbol_indexes
		248	WHERE id = ? AND state IN ('building', 'retired')`, id); err != nil {
		249	return err
		250	}
		251	}
		252	return nil
		253	}
		254
		255	// ReplaceSymbolIndex builds, publishes and purges in one call: the
		256	// worker's sequence, for a caller holding every row already.
		257	func (s *Store) ReplaceSymbolIndex(x SymbolIndex, syms []SymbolRow) (int64, error) {
		258	id, err := s.BeginSymbolIndex(x.RepoID, x.Commit, x.Tree)
		259	if err != nil {
		260	return 0, err
		261	}
		262	for len(syms) > 0 {
		263	n := min(len(syms), 5000)
		264	if err := s.AddSymbols(id, syms[:n]); err != nil {
140	return 0, err	265	return 0, err
141	}	266	}
142	syms = syms[n:]	267	syms = syms[n:]
143	}	268	}
144	return id, tx.Commit()	269	x.ID = id
		270	if err := s.PublishSymbolIndex(x); err != nil {
		271	return 0, err
		272	}
		273	return id, s.PurgeSymbolIndexes(x.RepoID)
		274	}
		275
		276	// RecordSymbolFailure records a run that could not build an index. The
		277	// current index, if any, stays current.
		278	func (s *Store) RecordSymbolFailure(repoID int64, tree, note string) error {
		279	_, err := s.DB.Exec(`
		280	INSERT INTO symbol_failures (repo_id, tree, note) VALUES (?, ?, ?)
		281	ON CONFLICT (repo_id) DO UPDATE SET tree = excluded.tree, note = excluded.note,
		282	failed_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')`, repoID, tree, note)
		283	return err
		284	}
		285
		286	// SymbolFailureFor returns the last failure, or ErrNotFound.
		287	func (s *Store) SymbolFailureFor(repoID int64) (SymbolFailure, error) {
		288	var f SymbolFailure
		289	err := s.DB.QueryRow(`SELECT tree, note, failed_at FROM symbol_failures WHERE repo_id = ?`, repoID).
		290	Scan(&f.Tree, &f.Note, &f.FailedAt)
		291	if errors.Is(err, sql.ErrNoRows) {
		292	return f, ErrNotFound
		293	}
		294	return f, err
		295	}
		296
		297	// SymbolFailureRecent reports whether tree failed within the last
		298	// seconds.
		299	func (s *Store) SymbolFailureRecent(repoID int64, tree string, seconds int) (bool, error) {
		300	var n int
		301	err := s.DB.QueryRow(`SELECT COUNT(*) FROM symbol_failures
		302	WHERE repo_id = ? AND tree = ?
		303	AND failed_at > strftime('%Y-%m-%dT%H:%M:%fZ', 'now', ?)`,
		304	repoID, tree, fmt.Sprintf("-%d seconds", seconds)).Scan(&n)
		305	return n > 0, err
145	}	306	}
146		307
147	// SearchSymbols finds the symbols whose name or key starts with q, ignoring	308	// SearchSymbols finds the symbols whose name or key starts with q, ignoring

internal/symbols/extract.go +16 −3

 // convention for generated files.
 var generatedGo = regexp.MustCompile(`(?m)^// Code generated .* DO NOT EDIT\.$`)
+// MaxNameBytes is the longest name the index keeps. A longer one is
+// dropped: no real definition is named that way, and a file of them is a
+// way to fill the index.
+const MaxNameBytes = 256
 // Extract returns the definitions in one file, in line order.
 func Extract(p string, data []byte) []Symbol {
+	var syms []Symbol
 	switch lang := Language(p); lang {
 	case "":
 		return nil
 	case "go":
-		return extractGo(data)
+		syms = extractGo(data)
 	case "org", "markdown":
-		return extractHeadings(lang, data)
+		syms = extractHeadings(lang, data)
 	default:
-		return extractLines(rules[lang], data)
+		syms = extractLines(rules[lang], data)
+	}
+	kept := syms[:0]
+	for _, s := range syms {
+		if len(s.Name) <= MaxNameBytes && len(s.Key) <= MaxNameBytes {
+			kept = append(kept, s)
+		}
 	}
+	return kept
 }
 func extractGo(data []byte) []Symbol {

internal/symbols/worker.go +101 −29

 	"gitbay.org/gitbay/internal/store"
 )
-// Bounds on one index run.
+// Bounds on one index run, and how it writes.
 const (
 	// DefaultMaxSymbols is where a repository's index stops growing.
 	DefaultMaxSymbols = 200_000
+	// DefaultMaxBytes bounds the names, keys and paths one index holds, so
+	// a tree of long names cannot fill the database within the count.
+	DefaultMaxBytes = 32 << 20
 	// DefaultMaxTime is how long one run may take.
 	DefaultMaxTime = 2 * time.Minute
+	// DefaultChunkRows is how many symbols one write transaction carries.
+	DefaultChunkRows = 5000
+	// DefaultBackoff is how long a failed tree waits before it is tried
+	// again.
+	DefaultBackoff = time.Hour
 )
 // Worker builds the index for each repository that has asked for one:
 	RepoDir    func(owner, name string) string
 	Tick       time.Duration
 	MaxSymbols int
+	MaxBytes   int
 	MaxTime    time.Duration
+	ChunkRows  int
+	Backoff    time.Duration
+	// chunkHook runs after each chunk is written; tests read the index
+	// mid-build through it.
+	chunkHook func(indexID int64)
 }
 func New(st *store.Store, repoDir func(owner, name string) string) *Worker {
 			tick = d
 		}
 	}
+	return NewWith(st, repoDir, tick)
+}
+// NewWith is New with the tick given, and every bound at its default.
+func NewWith(st *store.Store, repoDir func(owner, name string) string, tick time.Duration) *Worker {
 	return &Worker{St: st, RepoDir: repoDir, Tick: tick,
-		MaxSymbols: DefaultMaxSymbols, MaxTime: DefaultMaxTime}
+		MaxSymbols: DefaultMaxSymbols, MaxBytes: DefaultMaxBytes, MaxTime: DefaultMaxTime,
+		ChunkRows: DefaultChunkRows, Backoff: DefaultBackoff}
 }
 // Run sweeps until ctx ends.
 	}
 }
-// Sweep handles every waiting request once. A request is cleared whatever
+// Sweep handles every due request once. A request is cleared whatever
-// the outcome, so a repository whose index fails is not retried until
+// the outcome, except that a first failure is kept for one retry after
-// it is asked for again.
+// Backoff: a failure is retried once, not in a loop, and again only when
+// something asks.
 func (w *Worker) Sweep(ctx context.Context) {
 	reqs, err := w.St.SymbolRequests()
 	if err != nil {
 		if ctx.Err() != nil {
 			return
 		}
-		err := w.Index(ctx, req.RepoID, req.Force)
+		failed, err := w.Index(ctx, req.RepoID, req.Force)
 		if ctx.Err() != nil {
 			return // shutting down: the request stays for the next start
 		}
 		if err != nil {
 			slog.Warn("symbols: indexing", "repo", req.RepoID, "err", err)
 		}
+		if failed && req.Attempts == 0 {
+			w.St.DeferSymbolRequest(req, int(w.Backoff.Seconds()))
+			continue
+		}
 		w.St.DoneSymbolRequest(req)
 	}
 }
 // Index brings one repository's index up to its default branch's head.
-// A head whose tree is already indexed is left alone unless force is set.
+// A head whose tree is already indexed is left alone unless force is set,
-// The error is for the log: an index that could not be built is recorded
+// and so is one whose tree failed within Backoff. The new index is
-// as failed, and one cut short by a bound as partial.
+// written in chunks while no read can see it, then published in one
-func (w *Worker) Index(ctx context.Context, repoID int64, force bool) error {
+// short transaction. A run cut short by a bound publishes what it found
+// as partial; one that cannot build records the failure and leaves the
+// current index in place, reporting failed. The error is for the log.
+func (w *Worker) Index(ctx context.Context, repoID int64, force bool) (failed bool, err error) {
 	repo, err := w.St.RepoByID(repoID)
 	if errors.Is(err, store.ErrNotFound) {
-		return nil
+		return false, nil
 	} else if err != nil {
-		return err
+		return false, err
+	}
+	// A run that stopped part way left its building index behind.
+	if err := w.St.PurgeSymbolIndexes(repo.ID); err != nil {
+		return false, err
 	}
 	dir := w.RepoDir(repo.OwnerName, repo.Name)
 	commit, err := gitutil.ResolveRef(dir, "refs/heads/"+repo.DefaultBranch)
 	if err != nil {
-		return nil // no default branch yet: nothing to index
+		return false, nil // no default branch yet: nothing to index
 	}
 	tree, err := gitutil.ResolveTree(dir, commit)
 	if err != nil {
-		return err
+		return false, err
 	}
-	if cur, err := w.St.SymbolIndexFor(repo.ID); err == nil && cur.Tree == tree && !force {
+	if !force {
-		return nil
+		if cur, err := w.St.SymbolIndexFor(repo.ID); err == nil && cur.Tree == tree {
+			return false, nil
+		}
+		if recent, err := w.St.SymbolFailureRecent(repo.ID, tree, int(w.Backoff.Seconds())); err != nil || recent {
+			return false, err
+		}
 	}
 	x := store.SymbolIndex{RepoID: repo.ID, Commit: commit, Tree: tree, State: "ok"}
 	syms, files, runErr := w.collect(ctx, dir, tree)
 	if ctx.Err() != nil {
-		return ctx.Err()
+		return false, ctx.Err()
 	}
 	x.Files = files
 	switch {
 	case errors.Is(runErr, errSymbolCap):
 		x.State, x.Note = "partial", fmt.Sprintf("stopped at %d symbols", w.MaxSymbols)
+	case errors.Is(runErr, errByteBudget):
+		x.State, x.Note = "partial", fmt.Sprintf("stopped at %d bytes of names and paths", w.MaxBytes)
 	case errors.Is(runErr, context.DeadlineExceeded):
 		x.State, x.Note = "partial", fmt.Sprintf("stopped after %s", w.MaxTime)
 	case runErr != nil:
-		x.State, x.Note, syms = "failed", runErr.Error(), nil
+		if err := w.St.RecordSymbolFailure(repo.ID, tree, runErr.Error()); err != nil {
+			return true, err
+		}
+		return true, fmt.Errorf("%s failed: %v", repo.Path(), runErr)
+	}
+	if x.ID, err = w.St.BeginSymbolIndex(repo.ID, commit, tree); err != nil {
+		return false, err
 	}
-	if _, err := w.St.ReplaceSymbolIndex(x, syms); err != nil {
+	for len(syms) > 0 {
-		return err
+		if ctx.Err() != nil {
+			return false, ctx.Err() // the next run purges what was written
+		}
+		n := min(len(syms), w.ChunkRows)
+		if err := w.St.AddSymbols(x.ID, syms[:n]); err != nil {
+			return false, err
+		}
+		syms = syms[n:]
+		if w.chunkHook != nil {
+			w.chunkHook(x.ID)
+		}
+	}
+	if err := w.St.PublishSymbolIndex(x); err != nil {
+		return false, err
+	}
+	if err := w.St.PurgeSymbolIndexes(repo.ID); err != nil {
+		return false, err
 	}
 	if x.State != "ok" {
-		return fmt.Errorf("%s %s: %s", repo.Path(), x.State, x.Note)
+		return false, fmt.Errorf("%s %s: %s", repo.Path(), x.State, x.Note)
 	}
-	return nil
+	return false, nil
 }
-var errSymbolCap = errors.New("symbol cap reached")
+var (
+	errSymbolCap  = errors.New("symbol cap reached")
+	errByteBudget = errors.New("byte budget reached")
+)
 // collect reads every indexable blob in tree and extracts its symbols,
-// stopping at the symbol cap or the time bound with what it has.
+// stopping at the symbol cap, the byte budget or the time bound with what
+// it has.
 func (w *Worker) collect(ctx context.Context, dir, tree string) ([]store.SymbolRow, int, error) {
 	ctx, cancel := context.WithTimeout(ctx, w.MaxTime)
 	defer cancel()
 		shas = append(shas, b.SHA)
 	}
 	var out []store.SymbolRow
-	files := 0
+	files, used := 0, 0
-	capped := false
+	var stop error
 	err = gitutil.CatBlobs(ctx, dir, shas, func(i int, data []byte) bool {
 		if gitutil.IsBinary(data) {
 			return true
 		files++
 		for _, s := range Extract(paths[i], data) {
 			if len(out) >= w.MaxSymbols {
-				capped = true
+				stop = errSymbolCap
+				return false
+			}
+			// The key is counted too: it is stored beside the name.
+			n := len(s.Name) + len(s.Key) + len(paths[i])
+			if used+n > w.MaxBytes {
+				stop = errByteBudget
 				return false
 			}
+			used += n
 			out = append(out, store.SymbolRow{Name: s.Name, Key: s.Key, Kind: s.Kind, Path: paths[i], Line: s.Line})
 		}
 		return true
 	})
-	if capped {
+	if stop != nil {
-		return out, files, errSymbolCap
+		return out, files, stop
 	}
 	if err != nil && ctx.Err() == nil {
 		return nil, files, err

internal/symbols/worker_test.go +168 −12

	@@ -2,6 +2,7 @@ package symbols
2		2
3	import (	3	import (
4	"context"	4	"context"
		5	"fmt"
5	"os"	6	"os"
6	"os/exec"	7	"os/exec"
7	"path/filepath"	8	"path/filepath"
	@@ -45,8 +46,7 @@ func newFixture(t testing.T) fixture {
45	}	46	}
46	root := t.TempDir()	47	root := t.TempDir()
47	f := &fixture{t: t, st: st, repo: repo, src: filepath.Join(root, "src"), bare: filepath.Join(root, "app.git")}	48	f := &fixture{t: t, st: st, repo: repo, src: filepath.Join(root, "src"), bare: filepath.Join(root, "app.git")}
48	f.w = &Worker{St: st, RepoDir: func(owner, name string) string { return f.bare },	49	f.w = NewWith(st, func(owner, name string) string { return f.bare }, 0)
49	MaxSymbols: DefaultMaxSymbols, MaxTime: DefaultMaxTime}
50	f.git(root, "init", "-q", "-b", repo.DefaultBranch, "src")	50	f.git(root, "init", "-q", "-b", repo.DefaultBranch, "src")
51	f.git(root, "init", "-q", "--bare", f.bare)	51	f.git(root, "init", "-q", "--bare", f.bare)
52	return f	52	return f
	@@ -202,22 +202,178 @@ func TestWorkerTimeBound(t *testing.T) {
202	}	202	}
203	}	203	}
204		204
205	// A tree that cannot be read is recorded as failed, and the record stands	205	// breakBlob deletes the loose object of path at the default branch, so
206	// for that tree: asking again does not retry it.	206	// the tree cannot be read.
207	func TestWorkerFailureIsRecordedNotRetried(t *testing.T) {	207	func (f *fixture) breakBlob(path string) {
		208	f.t.Helper()
		209	blob := f.git(f.bare, "rev-parse", "refs/heads/"+f.repo.DefaultBranch+":"+path)
		210	if err := os.Remove(filepath.Join(f.bare, "objects", blob[:2], blob[2:])); err != nil {
		211	f.t.Fatal(err)
		212	}
		213	}
		214
		215	// A tree that cannot be read is recorded as a failure beside the current
		216	// index, which stays current. The request is kept for one retry after the
		217	// backoff rather than tried again at once; a new request for the same
		218	// tree inside the backoff is not retried either.
		219	func TestWorkerFailureKeepsCurrentIndex(t *testing.T) {
208	f := newFixture(t)	220	f := newFixture(t)
209	f.commit(map[string]string{"a.go": "package a\n\nfunc A() {}\n"})	221	f.commit(map[string]string{"a.go": "package a\n\nfunc A() {}\n"})
210	blob := f.git(f.bare, "rev-parse", "refs/heads/"+f.repo.DefaultBranch+":a.go")	222	good := f.sweep(false)
211	if err := os.Remove(filepath.Join(f.bare, "objects", blob[:2], blob[2:])); err != nil {	223
		224	f.commit(map[string]string{"b.go": "package a\n\nfunc B() {}\n"})
		225	f.breakBlob("b.go")
		226	f.st.RequestSymbolIndex(f.repo.ID, false)
		227	f.w.Sweep(context.Background())
		228	cur, err := f.st.SymbolIndexFor(f.repo.ID)
		229	if err != nil \|\| cur.ID != good.ID {
		230	t.Fatalf("current index after a failure = %+v, %v; want %d", cur, err, good.ID)
		231	}
		232	fail, err := f.st.SymbolFailureFor(f.repo.ID)
		233	if err != nil \|\| !strings.Contains(fail.Note, "b.go") {
		234	t.Fatalf("failure = %+v, %v", fail, err)
		235	}
		236	if reqs, _ := f.st.SymbolRequests(); len(reqs) != 0 {
		237	t.Fatalf("a failed request is due again at once: %+v", reqs)
		238	}
		239	var attempts int
		240	f.st.DB.QueryRow("SELECT attempts FROM symbol_requests WHERE repo_id = ?", f.repo.ID).Scan(&attempts)
		241	if attempts != 1 {
		242	t.Fatalf("attempts = %d, want the request kept for one retry", attempts)
		243	}
		244
		245	// Asked again inside the backoff: the failed tree is left alone.
		246	f.st.RequestSymbolIndex(f.repo.ID, false)
		247	failed, err := f.w.Index(context.Background(), f.repo.ID, false)
		248	if failed \|\| err != nil {
		249	t.Fatalf("a recently failed tree was retried: %v, %v", failed, err)
		250	}
		251
		252	// After the backoff it is tried again, and still fails.
		253	f.w.Backoff = 0
		254	if failed, _ := f.w.Index(context.Background(), f.repo.ID, false); !failed {
		255	t.Fatal("a failed tree was not retried after the backoff")
		256	}
		257
		258	// A second failure of a retry is the end of it.
		259	f.st.DB.Exec("DELETE FROM symbol_requests")
		260	f.st.DB.Exec("INSERT INTO symbol_requests (repo_id, attempts) VALUES (?, 1)", f.repo.ID)
		261	f.w.Sweep(context.Background())
		262	var n int
		263	f.st.DB.QueryRow("SELECT COUNT(*) FROM symbol_requests").Scan(&n)
		264	if n != 0 {
		265	t.Fatal("a failed retry was kept for another")
		266	}
		267
		268	// A push that changes the tree builds, and clears the failure.
		269	f.git(f.src, "rm", "-q", "b.go")
		270	f.commit(map[string]string{"c.go": "package a\n\nfunc C() {}\n"})
		271	next := f.sweep(false)
		272	if next.ID == good.ID {
		273	t.Fatal("a new tree after a failure was not indexed")
		274	}
		275	if _, err := f.st.SymbolFailureFor(f.repo.ID); err != store.ErrNotFound {
		276	t.Fatalf("failure kept after a good index: %v", err)
		277	}
		278	}
		279
		280	// A new index is written in chunks no read sees: throughout the build the
		281	// old index is current, and the flip replaces it whole.
		282	func TestWorkerBuildIsInvisibleUntilPublished(t *testing.T) {
		283	f := newFixture(t)
		284	f.commit(map[string]string{"a.go": "package a\n\nfunc Old() {}\n"})
		285	old := f.sweep(false)
		286
		287	f.commit(map[string]string{"a.go": "package a\n\nfunc New1() {}\nfunc New2() {}\nfunc New3() {}\n"})
		288	f.w.ChunkRows = 1
		289	chunks := 0
		290	f.w.chunkHook = func(building int64) {
		291	chunks++
		292	cur, err := f.st.SymbolIndexFor(f.repo.ID)
		293	if err != nil \|\| cur.ID != old.ID {
		294	t.Errorf("mid-build current index = %+v, %v; want the old one", cur, err)
		295	}
		296	if got := f.names(cur); len(got) != 1 \|\| got[0] != "a.go:Old" {
		297	t.Errorf("mid-build reads %v", got)
		298	}
		299	if b, _ := f.st.SymbolIndexByID(building); b.State != "building" {
		300	t.Errorf("index being built is %q", b.State)
		301	}
		302	}
		303	next := f.sweep(false)
		304	if chunks != 3 {
		305	t.Fatalf("%d chunks, want one per row", chunks)
		306	}
		307	if got := f.names(next); len(got) != 3 {
		308	t.Fatalf("after the flip: %v", got)
		309	}
		310	var indexes, rows int
		311	f.st.DB.QueryRow("SELECT COUNT(*) FROM symbol_indexes").Scan(&indexes)
		312	f.st.DB.QueryRow("SELECT COUNT(*) FROM symbols").Scan(&rows)
		313	if indexes != 1 \|\| rows != 3 {
		314	t.Fatalf("%d indexes and %d rows left, want the new index alone", indexes, rows)
		315	}
		316	}
		317
		318	// A run that stopped mid-build leaves the old index current, and the next
		319	// run deletes what it wrote.
		320	func TestWorkerCleansUpInterruptedBuild(t *testing.T) {
		321	f := newFixture(t)
		322	f.commit(map[string]string{"a.go": "package a\n\nfunc Old() {}\n"})
		323	old := f.sweep(false)
		324
		325	// What a crash between chunks leaves.
		326	orphan, err := f.st.BeginSymbolIndex(f.repo.ID, "c", "t")
		327	if err != nil {
		328	t.Fatal(err)
		329	}
		330	if err := f.st.AddSymbols(orphan, []store.SymbolRow{{Name: "Half", Key: "Half", Kind: "function", Path: "x.go", Line: 1}}); err != nil {
212	t.Fatal(err)	331	t.Fatal(err)
213	}	332	}
		333	if cur, _ := f.st.SymbolIndexFor(f.repo.ID); cur.ID != old.ID {
		334	t.Fatalf("an unfinished build became current: %+v", cur)
		335	}
		336
		337	f.commit(nil) // same tree: nothing to build, but the leftovers go
		338	f.sweep(false)
		339	if _, err := f.st.SymbolIndexByID(orphan); err != store.ErrNotFound {
		340	t.Fatalf("interrupted build still there: %v", err)
		341	}
		342	var rows int
		343	f.st.DB.QueryRow("SELECT COUNT(*) FROM symbols WHERE index_id = ?", orphan).Scan(&rows)
		344	if rows != 0 {
		345	t.Fatalf("%d rows of the interrupted build left", rows)
		346	}
		347	}
		348
		349	// A hostile tree: names past the length cap are dropped, and long
		350	// headings stop at the byte budget with a partial index saying so.
		351	func TestWorkerHostileNames(t *testing.T) {
		352	f := newFixture(t)
		353	files := map[string]string{}
		354	huge := strings.Repeat("x", MaxNameBytes+1)
		355	for i := 0; i < 20; i++ {
		356	files[fmt.Sprintf("huge%02d.go", i)] = "package a\n\nfunc " + huge + "() {}\nvar " + huge + " int\n"
		357	}
		358	var md strings.Builder
		359	for i := 0; i < 500; i++ {
		360	fmt.Fprintf(&md, "# %03d %s\n", i, strings.Repeat("h", 200))
		361	}
		362	files["notes.md"] = md.String()
		363	f.commit(files)
		364	f.w.MaxBytes = 20_000
214	x := f.sweep(false)	365	x := f.sweep(false)
215	if x.State != "failed" \|\| x.Note == "" \|\| x.Symbols != 0 {	366	if x.State != "partial" \|\| !strings.Contains(x.Note, "20000 bytes") {
216	t.Fatalf("index = %+v, want failed with a note", x)	367	t.Fatalf("index = %+v, want partial at the byte budget", x)
		368	}
		369	var longest, total int
		370	f.st.DB.QueryRow("SELECT COALESCE(MAX(length(name)), 0), COALESCE(SUM(length(name) + length(key) + length(path)), 0) FROM symbols").
		371	Scan(&longest, &total)
		372	if longest > MaxNameBytes {
		373	t.Errorf("a %d-byte name was kept", longest)
217	}	374	}
218	again := f.sweep(false)	375	if total > 20_000 \|\| x.Symbols == 0 {
219	if again.ID != x.ID {	376	t.Errorf("%d symbols holding %d bytes, want some within the budget", x.Symbols, total)
220	t.Fatalf("a failed tree was retried: %+v then %+v", x, again)
221	}	377	}
222	}	378	}
223		379

Commit c528009533

Commit `c528009533`