internal/store/symbols.go

  1package store
  2
  3import (
  4	"database/sql"
  5	"errors"
  6	"fmt"
  7	"strings"
  8	"unicode/utf8"
  9)
 10
 11// SymbolIndex is one symbol index of a repository. The states are
 12// described with the table's schema; reads only ever see ok or partial.
 13type SymbolIndex struct {
 14	ID      int64
 15	RepoID  int64
 16	Commit  string
 17	Tree    string
 18	State   string // building | ok | partial | retired
 19	Note    string
 20	Files   int
 21	Symbols int
 22	BuiltAt string
 23}
 24
 25// SymbolFailure is the last run that could not build an index.
 26type SymbolFailure struct {
 27	Tree     string
 28	Note     string
 29	FailedAt string
 30}
 31
 32// SymbolRow is one definition. Key is the name as written where it is
 33// used; see internal/symbols.
 34type SymbolRow struct {
 35	ID   int64
 36	Name string
 37	Key  string
 38	Kind string
 39	Path string
 40	Line int
 41}
 42
 43// SymbolTarget is where a key is defined: Count definitions, the first of
 44// them at Path and Line.
 45type SymbolTarget struct {
 46	Count int
 47	Path  string
 48	Line  int
 49}
 50
 51// SymbolRequest is a repository waiting for the index worker.
 52type SymbolRequest struct {
 53	RepoID   int64
 54	Seq      int64
 55	Force    bool
 56	Attempts int
 57}
 58
 59// RequestSymbolIndex queues a repository for the index worker. A request
 60// already waiting is bumped, so one taken by a running build is not
 61// cleared when that build ends, and a deferred retry becomes due now;
 62// force is kept once set.
 63func (s *Store) RequestSymbolIndex(repoID int64, force bool) error {
 64	_, err := s.DB.Exec(`
 65		INSERT INTO symbol_requests (repo_id, force) VALUES (?, ?)
 66		ON CONFLICT (repo_id) DO UPDATE SET seq = seq + 1,
 67		       force = MAX(force, excluded.force), attempts = 0, not_before = '',
 68		       requested_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')`, repoID, force)
 69	return err
 70}
 71
 72// SymbolRequests lists the requests that are due, oldest first.
 73func (s *Store) SymbolRequests() ([]SymbolRequest, error) {
 74	rows, err := s.DB.Query(`
 75		SELECT repo_id, seq, force, attempts FROM symbol_requests
 76		WHERE not_before = '' OR not_before <= strftime('%Y-%m-%dT%H:%M:%fZ','now')
 77		ORDER BY requested_at, repo_id`)
 78	if err != nil {
 79		return nil, err
 80	}
 81	defer rows.Close()
 82	var out []SymbolRequest
 83	for rows.Next() {
 84		var r SymbolRequest
 85		if err := rows.Scan(&r.RepoID, &r.Seq, &r.Force, &r.Attempts); err != nil {
 86			return nil, err
 87		}
 88		out = append(out, r)
 89	}
 90	return out, rows.Err()
 91}
 92
 93// DoneSymbolRequest clears a request the worker has handled, unless it was
 94// requested again since it was read.
 95func (s *Store) DoneSymbolRequest(r SymbolRequest) error {
 96	_, err := s.DB.Exec("DELETE FROM symbol_requests WHERE repo_id = ? AND seq = ?", r.RepoID, r.Seq)
 97	return err
 98}
 99
100// DeferSymbolRequest keeps a request for another attempt after seconds,
101// unless it was requested again since it was read.
102func (s *Store) DeferSymbolRequest(r SymbolRequest, seconds int) error {
103	_, err := s.DB.Exec(`
104		UPDATE symbol_requests SET force = 0, attempts = attempts + 1,
105		       not_before = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', ?)
106		WHERE repo_id = ? AND seq = ?`, fmt.Sprintf("+%d seconds", seconds), r.RepoID, r.Seq)
107	return err
108}
109
110const symbolIndexCols = `id, repo_id, commit_sha, tree, state, note, files, symbols, built_at`
111
112func scanSymbolIndex(row *sql.Row) (SymbolIndex, error) {
113	var x SymbolIndex
114	err := row.Scan(&x.ID, &x.RepoID, &x.Commit, &x.Tree, &x.State, &x.Note, &x.Files, &x.Symbols, &x.BuiltAt)
115	if errors.Is(err, sql.ErrNoRows) {
116		return x, ErrNotFound
117	}
118	return x, err
119}
120
121// SymbolIndexFor returns a repository's current index, or ErrNotFound
122// when none has been published.
123func (s *Store) SymbolIndexFor(repoID int64) (SymbolIndex, error) {
124	return scanSymbolIndex(s.DB.QueryRow(`SELECT `+symbolIndexCols+`
125		FROM symbol_indexes WHERE repo_id = ? AND state IN ('ok', 'partial')`, repoID))
126}
127
128// SymbolIndexByID returns an index in any state.
129func (s *Store) SymbolIndexByID(id int64) (SymbolIndex, error) {
130	return scanSymbolIndex(s.DB.QueryRow(`SELECT `+symbolIndexCols+`
131		FROM symbol_indexes WHERE id = ?`, id))
132}
133
134// BeginSymbolIndex creates an index in the building state, which no read
135// sees until PublishSymbolIndex.
136func (s *Store) BeginSymbolIndex(repoID int64, commit, tree string) (int64, error) {
137	res, err := s.DB.Exec(`
138		INSERT INTO symbol_indexes (repo_id, commit_sha, tree, state)
139		VALUES (?, ?, ?, 'building')`, repoID, commit, tree)
140	if err != nil {
141		return 0, err
142	}
143	return res.LastInsertId()
144}
145
146// symbolInsertRows is how many rows one INSERT carries: eight columns
147// each, well under SQLite's variable limit.
148const symbolInsertRows = 200
149
150// AddSymbols writes rows into an index being built, in one transaction.
151// The caller keeps each call to a few thousand rows so the write lock is
152// held briefly.
153func (s *Store) AddSymbols(indexID int64, syms []SymbolRow) error {
154	tx, err := s.DB.Begin()
155	if err != nil {
156		return err
157	}
158	defer tx.Rollback()
159	for len(syms) > 0 {
160		n := min(len(syms), symbolInsertRows)
161		var q strings.Builder
162		q.WriteString("INSERT INTO symbols (index_id, name, lname, key, lkey, kind, path, line) VALUES ")
163		args := make([]any, 0, n*8)
164		for i, r := range syms[:n] {
165			if i > 0 {
166				q.WriteString(",")
167			}
168			q.WriteString("(?,?,?,?,?,?,?,?)")
169			args = append(args, indexID, r.Name, strings.ToLower(r.Name), r.Key, strings.ToLower(r.Key), r.Kind, r.Path, r.Line)
170		}
171		if _, err := tx.Exec(q.String(), args...); err != nil {
172			return err
173		}
174		syms = syms[n:]
175	}
176	return tx.Commit()
177}
178
179// PublishSymbolIndex makes a built index the repository's current one, as
180// ok or partial, and retires the one it replaces, in one short
181// transaction. A recorded failure is cleared.
182func (s *Store) PublishSymbolIndex(x SymbolIndex) error {
183	tx, err := s.DB.Begin()
184	if err != nil {
185		return err
186	}
187	defer tx.Rollback()
188	if _, err := tx.Exec(`
189		UPDATE symbol_indexes SET state = 'retired'
190		WHERE repo_id = ? AND state IN ('ok', 'partial')`, x.RepoID); err != nil {
191		return err
192	}
193	res, err := tx.Exec(`
194		UPDATE symbol_indexes SET state = ?, note = ?, files = ?,
195		       symbols = (SELECT COUNT(*) FROM symbols WHERE index_id = ?),
196		       built_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')
197		WHERE id = ? AND repo_id = ? AND state = 'building'`,
198		x.State, x.Note, x.Files, x.ID, x.ID, x.RepoID)
199	if err != nil {
200		return err
201	}
202	if n, _ := res.RowsAffected(); n != 1 {
203		return fmt.Errorf("symbol index %d is not being built", x.ID)
204	}
205	if _, err := tx.Exec("DELETE FROM symbol_failures WHERE repo_id = ?", x.RepoID); err != nil {
206		return err
207	}
208	return tx.Commit()
209}
210
211// symbolDeleteRows is how many symbols one purge transaction deletes.
212const symbolDeleteRows = 5000
213
214// PurgeSymbolIndexes deletes a repository's indexes that are not current:
215// retired ones, and building ones left by a run that did not finish. The
216// symbols go a chunk per transaction, then the index row.
217func (s *Store) PurgeSymbolIndexes(repoID int64) error {
218	rows, err := s.DB.Query(`SELECT id FROM symbol_indexes
219		WHERE repo_id = ? AND state IN ('building', 'retired')`, repoID)
220	if err != nil {
221		return err
222	}
223	var ids []int64
224	for rows.Next() {
225		var id int64
226		if err := rows.Scan(&id); err != nil {
227			rows.Close()
228			return err
229		}
230		ids = append(ids, id)
231	}
232	rows.Close()
233	if err := rows.Err(); err != nil {
234		return err
235	}
236	for _, id := range ids {
237		for {
238			res, err := s.DB.Exec(`DELETE FROM symbols WHERE id IN
239				(SELECT id FROM symbols WHERE index_id = ? LIMIT ?)`, id, symbolDeleteRows)
240			if err != nil {
241				return err
242			}
243			if n, _ := res.RowsAffected(); n == 0 {
244				break
245			}
246		}
247		if _, err := s.DB.Exec(`DELETE FROM symbol_indexes
248			WHERE id = ? AND state IN ('building', 'retired')`, id); err != nil {
249			return err
250		}
251	}
252	return nil
253}
254
255// ReplaceSymbolIndex builds, publishes and purges in one call: the
256// worker's sequence, for a caller holding every row already.
257func (s *Store) ReplaceSymbolIndex(x SymbolIndex, syms []SymbolRow) (int64, error) {
258	id, err := s.BeginSymbolIndex(x.RepoID, x.Commit, x.Tree)
259	if err != nil {
260		return 0, err
261	}
262	for len(syms) > 0 {
263		n := min(len(syms), 5000)
264		if err := s.AddSymbols(id, syms[:n]); err != nil {
265			return 0, err
266		}
267		syms = syms[n:]
268	}
269	x.ID = id
270	if err := s.PublishSymbolIndex(x); err != nil {
271		return 0, err
272	}
273	return id, s.PurgeSymbolIndexes(x.RepoID)
274}
275
276// RecordSymbolFailure records a run that could not build an index. The
277// current index, if any, stays current.
278func (s *Store) RecordSymbolFailure(repoID int64, tree, note string) error {
279	_, err := s.DB.Exec(`
280		INSERT INTO symbol_failures (repo_id, tree, note) VALUES (?, ?, ?)
281		ON CONFLICT (repo_id) DO UPDATE SET tree = excluded.tree, note = excluded.note,
282		       failed_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')`, repoID, tree, note)
283	return err
284}
285
286// SymbolFailureFor returns the last failure, or ErrNotFound.
287func (s *Store) SymbolFailureFor(repoID int64) (SymbolFailure, error) {
288	var f SymbolFailure
289	err := s.DB.QueryRow(`SELECT tree, note, failed_at FROM symbol_failures WHERE repo_id = ?`, repoID).
290		Scan(&f.Tree, &f.Note, &f.FailedAt)
291	if errors.Is(err, sql.ErrNoRows) {
292		return f, ErrNotFound
293	}
294	return f, err
295}
296
297// SymbolFailureRecent reports whether tree failed within the last
298// seconds.
299func (s *Store) SymbolFailureRecent(repoID int64, tree string, seconds int) (bool, error) {
300	var n int
301	err := s.DB.QueryRow(`SELECT COUNT(*) FROM symbol_failures
302		WHERE repo_id = ? AND tree = ?
303		  AND failed_at > strftime('%Y-%m-%dT%H:%M:%fZ', 'now', ?)`,
304		repoID, tree, fmt.Sprintf("-%d seconds", seconds)).Scan(&n)
305	return n > 0, err
306}
307
308// SearchSymbols finds the symbols whose name or key starts with q, ignoring
309// case, ranked: an exact match first, then a prefix match, each
310// case-sensitive before case-insensitive, then by name, path and line.
311// after is the id of the last row of the previous page, 0 for the first;
312// an id that is no longer in the result gives an empty page. kind filters
313// when not empty.
314func (s *Store) SearchSymbols(indexID int64, q, kind string, limit int, after int64) ([]SymbolRow, error) {
315	lq := strings.ToLower(q)
316	hi := lq + "\U0010FFFF"
317	n := utf8.RuneCountInString(q)
318	lim := -1
319	if limit > 0 {
320		lim = limit
321	}
322	rows, err := s.DB.Query(`
323		WITH m AS (
324			SELECT id, name, key, kind, path, line,
325			       CASE WHEN name = ? OR key = ? THEN 0
326			            WHEN substr(name, 1, ?) = ? OR substr(key, 1, ?) = ? THEN 1
327			            WHEN lname = ? OR lkey = ? THEN 2
328			            ELSE 3 END AS rank
329			FROM symbols
330			WHERE index_id = ?
331			  AND ((lname >= ? AND lname < ?) OR (lkey >= ? AND lkey < ?))
332			  AND (? = '' OR kind = ?)
333		)
334		SELECT id, name, key, kind, path, line FROM m
335		WHERE ? = 0 OR (rank, name, path, line, id) >
336		      (SELECT rank, name, path, line, id FROM m WHERE id = ?)
337		ORDER BY rank, name, path, line, id
338		LIMIT ?`,
339		q, q, n, q, n, q, lq, lq,
340		indexID, lq, hi, lq, hi, kind, kind,
341		after, after, lim)
342	if err != nil {
343		return nil, err
344	}
345	return scanSymbols(rows)
346}
347
348// SymbolsInFile lists one file's symbols in line order.
349func (s *Store) SymbolsInFile(indexID int64, path string) ([]SymbolRow, error) {
350	rows, err := s.DB.Query(`
351		SELECT id, name, key, kind, path, line FROM symbols
352		WHERE index_id = ? AND path = ? ORDER BY line, id`, indexID, path)
353	if err != nil {
354		return nil, err
355	}
356	return scanSymbols(rows)
357}
358
359func scanSymbols(rows *sql.Rows) ([]SymbolRow, error) {
360	defer rows.Close()
361	var out []SymbolRow
362	for rows.Next() {
363		var r SymbolRow
364		if err := rows.Scan(&r.ID, &r.Name, &r.Key, &r.Kind, &r.Path, &r.Line); err != nil {
365			return nil, err
366		}
367		out = append(out, r)
368	}
369	return out, rows.Err()
370}
371
372// symbolKeysPerQuery bounds the IN list of one SymbolTargets query.
373const symbolKeysPerQuery = 500
374
375// SymbolTargets looks up where each of keys is defined, case-sensitively.
376// A key with no definition is absent from the result.
377func (s *Store) SymbolTargets(indexID int64, keys []string) (map[string]SymbolTarget, error) {
378	out := map[string]SymbolTarget{}
379	for len(keys) > 0 {
380		n := min(len(keys), symbolKeysPerQuery)
381		args := make([]any, 0, n+1)
382		args = append(args, indexID)
383		for _, k := range keys[:n] {
384			args = append(args, k)
385		}
386		// SQLite takes the bare columns of an aggregate query with MIN
387		// from the row that holds the minimum: the first definition.
388		rows, err := s.DB.Query(`
389			SELECT key, COUNT(*), path, line, MIN(id) FROM symbols
390			WHERE index_id = ? AND key IN (?`+strings.Repeat(",?", n-1)+`)
391			GROUP BY key`, args...)
392		if err != nil {
393			return nil, err
394		}
395		for rows.Next() {
396			var k string
397			var t SymbolTarget
398			var id int64
399			if err := rows.Scan(&k, &t.Count, &t.Path, &t.Line, &id); err != nil {
400				rows.Close()
401				return nil, err
402			}
403			out[k] = t
404		}
405		rows.Close()
406		if err := rows.Err(); err != nil {
407			return nil, err
408		}
409		keys = keys[n:]
410	}
411	return out, nil
412}