internal/store/symbols.go
412 lines · 12768 bytes
1package store
2
3import (
4 "database/sql"
5 "errors"
6 "fmt"
7 "strings"
8 "unicode/utf8"
9)
10
11// SymbolIndex is one symbol index of a repository. The states are
12// described with the table's schema; reads only ever see ok or partial.
13type SymbolIndex struct {
14 ID int64
15 RepoID int64
16 Commit string
17 Tree string
18 State string // building | ok | partial | retired
19 Note string
20 Files int
21 Symbols int
22 BuiltAt string
23}
24
25// SymbolFailure is the last run that could not build an index.
26type SymbolFailure struct {
27 Tree string
28 Note string
29 FailedAt string
30}
31
32// SymbolRow is one definition. Key is the name as written where it is
33// used; see internal/symbols.
34type SymbolRow struct {
35 ID int64
36 Name string
37 Key string
38 Kind string
39 Path string
40 Line int
41}
42
43// SymbolTarget is where a key is defined: Count definitions, the first of
44// them at Path and Line.
45type SymbolTarget struct {
46 Count int
47 Path string
48 Line int
49}
50
51// SymbolRequest is a repository waiting for the index worker.
52type SymbolRequest struct {
53 RepoID int64
54 Seq int64
55 Force bool
56 Attempts int
57}
58
59// RequestSymbolIndex queues a repository for the index worker. A request
60// already waiting is bumped, so one taken by a running build is not
61// cleared when that build ends, and a deferred retry becomes due now;
62// force is kept once set.
63func (s *Store) RequestSymbolIndex(repoID int64, force bool) error {
64 _, err := s.DB.Exec(`
65 INSERT INTO symbol_requests (repo_id, force) VALUES (?, ?)
66 ON CONFLICT (repo_id) DO UPDATE SET seq = seq + 1,
67 force = MAX(force, excluded.force), attempts = 0, not_before = '',
68 requested_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')`, repoID, force)
69 return err
70}
71
72// SymbolRequests lists the requests that are due, oldest first.
73func (s *Store) SymbolRequests() ([]SymbolRequest, error) {
74 rows, err := s.DB.Query(`
75 SELECT repo_id, seq, force, attempts FROM symbol_requests
76 WHERE not_before = '' OR not_before <= strftime('%Y-%m-%dT%H:%M:%fZ','now')
77 ORDER BY requested_at, repo_id`)
78 if err != nil {
79 return nil, err
80 }
81 defer rows.Close()
82 var out []SymbolRequest
83 for rows.Next() {
84 var r SymbolRequest
85 if err := rows.Scan(&r.RepoID, &r.Seq, &r.Force, &r.Attempts); err != nil {
86 return nil, err
87 }
88 out = append(out, r)
89 }
90 return out, rows.Err()
91}
92
93// DoneSymbolRequest clears a request the worker has handled, unless it was
94// requested again since it was read.
95func (s *Store) DoneSymbolRequest(r SymbolRequest) error {
96 _, err := s.DB.Exec("DELETE FROM symbol_requests WHERE repo_id = ? AND seq = ?", r.RepoID, r.Seq)
97 return err
98}
99
100// DeferSymbolRequest keeps a request for another attempt after seconds,
101// unless it was requested again since it was read.
102func (s *Store) DeferSymbolRequest(r SymbolRequest, seconds int) error {
103 _, err := s.DB.Exec(`
104 UPDATE symbol_requests SET force = 0, attempts = attempts + 1,
105 not_before = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', ?)
106 WHERE repo_id = ? AND seq = ?`, fmt.Sprintf("+%d seconds", seconds), r.RepoID, r.Seq)
107 return err
108}
109
110const symbolIndexCols = `id, repo_id, commit_sha, tree, state, note, files, symbols, built_at`
111
112func scanSymbolIndex(row *sql.Row) (SymbolIndex, error) {
113 var x SymbolIndex
114 err := row.Scan(&x.ID, &x.RepoID, &x.Commit, &x.Tree, &x.State, &x.Note, &x.Files, &x.Symbols, &x.BuiltAt)
115 if errors.Is(err, sql.ErrNoRows) {
116 return x, ErrNotFound
117 }
118 return x, err
119}
120
121// SymbolIndexFor returns a repository's current index, or ErrNotFound
122// when none has been published.
123func (s *Store) SymbolIndexFor(repoID int64) (SymbolIndex, error) {
124 return scanSymbolIndex(s.DB.QueryRow(`SELECT `+symbolIndexCols+`
125 FROM symbol_indexes WHERE repo_id = ? AND state IN ('ok', 'partial')`, repoID))
126}
127
128// SymbolIndexByID returns an index in any state.
129func (s *Store) SymbolIndexByID(id int64) (SymbolIndex, error) {
130 return scanSymbolIndex(s.DB.QueryRow(`SELECT `+symbolIndexCols+`
131 FROM symbol_indexes WHERE id = ?`, id))
132}
133
134// BeginSymbolIndex creates an index in the building state, which no read
135// sees until PublishSymbolIndex.
136func (s *Store) BeginSymbolIndex(repoID int64, commit, tree string) (int64, error) {
137 res, err := s.DB.Exec(`
138 INSERT INTO symbol_indexes (repo_id, commit_sha, tree, state)
139 VALUES (?, ?, ?, 'building')`, repoID, commit, tree)
140 if err != nil {
141 return 0, err
142 }
143 return res.LastInsertId()
144}
145
146// symbolInsertRows is how many rows one INSERT carries: eight columns
147// each, well under SQLite's variable limit.
148const symbolInsertRows = 200
149
150// AddSymbols writes rows into an index being built, in one transaction.
151// The caller keeps each call to a few thousand rows so the write lock is
152// held briefly.
153func (s *Store) AddSymbols(indexID int64, syms []SymbolRow) error {
154 tx, err := s.DB.Begin()
155 if err != nil {
156 return err
157 }
158 defer tx.Rollback()
159 for len(syms) > 0 {
160 n := min(len(syms), symbolInsertRows)
161 var q strings.Builder
162 q.WriteString("INSERT INTO symbols (index_id, name, lname, key, lkey, kind, path, line) VALUES ")
163 args := make([]any, 0, n*8)
164 for i, r := range syms[:n] {
165 if i > 0 {
166 q.WriteString(",")
167 }
168 q.WriteString("(?,?,?,?,?,?,?,?)")
169 args = append(args, indexID, r.Name, strings.ToLower(r.Name), r.Key, strings.ToLower(r.Key), r.Kind, r.Path, r.Line)
170 }
171 if _, err := tx.Exec(q.String(), args...); err != nil {
172 return err
173 }
174 syms = syms[n:]
175 }
176 return tx.Commit()
177}
178
179// PublishSymbolIndex makes a built index the repository's current one, as
180// ok or partial, and retires the one it replaces, in one short
181// transaction. A recorded failure is cleared.
182func (s *Store) PublishSymbolIndex(x SymbolIndex) error {
183 tx, err := s.DB.Begin()
184 if err != nil {
185 return err
186 }
187 defer tx.Rollback()
188 if _, err := tx.Exec(`
189 UPDATE symbol_indexes SET state = 'retired'
190 WHERE repo_id = ? AND state IN ('ok', 'partial')`, x.RepoID); err != nil {
191 return err
192 }
193 res, err := tx.Exec(`
194 UPDATE symbol_indexes SET state = ?, note = ?, files = ?,
195 symbols = (SELECT COUNT(*) FROM symbols WHERE index_id = ?),
196 built_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')
197 WHERE id = ? AND repo_id = ? AND state = 'building'`,
198 x.State, x.Note, x.Files, x.ID, x.ID, x.RepoID)
199 if err != nil {
200 return err
201 }
202 if n, _ := res.RowsAffected(); n != 1 {
203 return fmt.Errorf("symbol index %d is not being built", x.ID)
204 }
205 if _, err := tx.Exec("DELETE FROM symbol_failures WHERE repo_id = ?", x.RepoID); err != nil {
206 return err
207 }
208 return tx.Commit()
209}
210
211// symbolDeleteRows is how many symbols one purge transaction deletes.
212const symbolDeleteRows = 5000
213
214// PurgeSymbolIndexes deletes a repository's indexes that are not current:
215// retired ones, and building ones left by a run that did not finish. The
216// symbols go a chunk per transaction, then the index row.
217func (s *Store) PurgeSymbolIndexes(repoID int64) error {
218 rows, err := s.DB.Query(`SELECT id FROM symbol_indexes
219 WHERE repo_id = ? AND state IN ('building', 'retired')`, repoID)
220 if err != nil {
221 return err
222 }
223 var ids []int64
224 for rows.Next() {
225 var id int64
226 if err := rows.Scan(&id); err != nil {
227 rows.Close()
228 return err
229 }
230 ids = append(ids, id)
231 }
232 rows.Close()
233 if err := rows.Err(); err != nil {
234 return err
235 }
236 for _, id := range ids {
237 for {
238 res, err := s.DB.Exec(`DELETE FROM symbols WHERE id IN
239 (SELECT id FROM symbols WHERE index_id = ? LIMIT ?)`, id, symbolDeleteRows)
240 if err != nil {
241 return err
242 }
243 if n, _ := res.RowsAffected(); n == 0 {
244 break
245 }
246 }
247 if _, err := s.DB.Exec(`DELETE FROM symbol_indexes
248 WHERE id = ? AND state IN ('building', 'retired')`, id); err != nil {
249 return err
250 }
251 }
252 return nil
253}
254
255// ReplaceSymbolIndex builds, publishes and purges in one call: the
256// worker's sequence, for a caller holding every row already.
257func (s *Store) ReplaceSymbolIndex(x SymbolIndex, syms []SymbolRow) (int64, error) {
258 id, err := s.BeginSymbolIndex(x.RepoID, x.Commit, x.Tree)
259 if err != nil {
260 return 0, err
261 }
262 for len(syms) > 0 {
263 n := min(len(syms), 5000)
264 if err := s.AddSymbols(id, syms[:n]); err != nil {
265 return 0, err
266 }
267 syms = syms[n:]
268 }
269 x.ID = id
270 if err := s.PublishSymbolIndex(x); err != nil {
271 return 0, err
272 }
273 return id, s.PurgeSymbolIndexes(x.RepoID)
274}
275
276// RecordSymbolFailure records a run that could not build an index. The
277// current index, if any, stays current.
278func (s *Store) RecordSymbolFailure(repoID int64, tree, note string) error {
279 _, err := s.DB.Exec(`
280 INSERT INTO symbol_failures (repo_id, tree, note) VALUES (?, ?, ?)
281 ON CONFLICT (repo_id) DO UPDATE SET tree = excluded.tree, note = excluded.note,
282 failed_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')`, repoID, tree, note)
283 return err
284}
285
286// SymbolFailureFor returns the last failure, or ErrNotFound.
287func (s *Store) SymbolFailureFor(repoID int64) (SymbolFailure, error) {
288 var f SymbolFailure
289 err := s.DB.QueryRow(`SELECT tree, note, failed_at FROM symbol_failures WHERE repo_id = ?`, repoID).
290 Scan(&f.Tree, &f.Note, &f.FailedAt)
291 if errors.Is(err, sql.ErrNoRows) {
292 return f, ErrNotFound
293 }
294 return f, err
295}
296
297// SymbolFailureRecent reports whether tree failed within the last
298// seconds.
299func (s *Store) SymbolFailureRecent(repoID int64, tree string, seconds int) (bool, error) {
300 var n int
301 err := s.DB.QueryRow(`SELECT COUNT(*) FROM symbol_failures
302 WHERE repo_id = ? AND tree = ?
303 AND failed_at > strftime('%Y-%m-%dT%H:%M:%fZ', 'now', ?)`,
304 repoID, tree, fmt.Sprintf("-%d seconds", seconds)).Scan(&n)
305 return n > 0, err
306}
307
308// SearchSymbols finds the symbols whose name or key starts with q, ignoring
309// case, ranked: an exact match first, then a prefix match, each
310// case-sensitive before case-insensitive, then by name, path and line.
311// after is the id of the last row of the previous page, 0 for the first;
312// an id that is no longer in the result gives an empty page. kind filters
313// when not empty.
314func (s *Store) SearchSymbols(indexID int64, q, kind string, limit int, after int64) ([]SymbolRow, error) {
315 lq := strings.ToLower(q)
316 hi := lq + "\U0010FFFF"
317 n := utf8.RuneCountInString(q)
318 lim := -1
319 if limit > 0 {
320 lim = limit
321 }
322 rows, err := s.DB.Query(`
323 WITH m AS (
324 SELECT id, name, key, kind, path, line,
325 CASE WHEN name = ? OR key = ? THEN 0
326 WHEN substr(name, 1, ?) = ? OR substr(key, 1, ?) = ? THEN 1
327 WHEN lname = ? OR lkey = ? THEN 2
328 ELSE 3 END AS rank
329 FROM symbols
330 WHERE index_id = ?
331 AND ((lname >= ? AND lname < ?) OR (lkey >= ? AND lkey < ?))
332 AND (? = '' OR kind = ?)
333 )
334 SELECT id, name, key, kind, path, line FROM m
335 WHERE ? = 0 OR (rank, name, path, line, id) >
336 (SELECT rank, name, path, line, id FROM m WHERE id = ?)
337 ORDER BY rank, name, path, line, id
338 LIMIT ?`,
339 q, q, n, q, n, q, lq, lq,
340 indexID, lq, hi, lq, hi, kind, kind,
341 after, after, lim)
342 if err != nil {
343 return nil, err
344 }
345 return scanSymbols(rows)
346}
347
348// SymbolsInFile lists one file's symbols in line order.
349func (s *Store) SymbolsInFile(indexID int64, path string) ([]SymbolRow, error) {
350 rows, err := s.DB.Query(`
351 SELECT id, name, key, kind, path, line FROM symbols
352 WHERE index_id = ? AND path = ? ORDER BY line, id`, indexID, path)
353 if err != nil {
354 return nil, err
355 }
356 return scanSymbols(rows)
357}
358
359func scanSymbols(rows *sql.Rows) ([]SymbolRow, error) {
360 defer rows.Close()
361 var out []SymbolRow
362 for rows.Next() {
363 var r SymbolRow
364 if err := rows.Scan(&r.ID, &r.Name, &r.Key, &r.Kind, &r.Path, &r.Line); err != nil {
365 return nil, err
366 }
367 out = append(out, r)
368 }
369 return out, rows.Err()
370}
371
372// symbolKeysPerQuery bounds the IN list of one SymbolTargets query.
373const symbolKeysPerQuery = 500
374
375// SymbolTargets looks up where each of keys is defined, case-sensitively.
376// A key with no definition is absent from the result.
377func (s *Store) SymbolTargets(indexID int64, keys []string) (map[string]SymbolTarget, error) {
378 out := map[string]SymbolTarget{}
379 for len(keys) > 0 {
380 n := min(len(keys), symbolKeysPerQuery)
381 args := make([]any, 0, n+1)
382 args = append(args, indexID)
383 for _, k := range keys[:n] {
384 args = append(args, k)
385 }
386 // SQLite takes the bare columns of an aggregate query with MIN
387 // from the row that holds the minimum: the first definition.
388 rows, err := s.DB.Query(`
389 SELECT key, COUNT(*), path, line, MIN(id) FROM symbols
390 WHERE index_id = ? AND key IN (?`+strings.Repeat(",?", n-1)+`)
391 GROUP BY key`, args...)
392 if err != nil {
393 return nil, err
394 }
395 for rows.Next() {
396 var k string
397 var t SymbolTarget
398 var id int64
399 if err := rows.Scan(&k, &t.Count, &t.Path, &t.Line, &id); err != nil {
400 rows.Close()
401 return nil, err
402 }
403 out[k] = t
404 }
405 rows.Close()
406 if err := rows.Err(); err != nil {
407 return nil, err
408 }
409 keys = keys[n:]
410 }
411 return out, nil
412}