internal/symbols/worker_test.go
390 lines · 12682 bytes
1package symbols
2
3import (
4 "context"
5 "fmt"
6 "os"
7 "os/exec"
8 "path/filepath"
9 "strings"
10 "testing"
11 "time"
12
13 "gitbay.org/gitbay/internal/store"
14)
15
16type fixture struct {
17 t *testing.T
18 st *store.Store
19 repo store.Repo
20 src string
21 bare string
22 w *Worker
23}
24
25func newFixture(t *testing.T) *fixture {
26 t.Helper()
27 st, err := store.Open(":memory:")
28 if err != nil {
29 t.Fatal(err)
30 }
31 t.Cleanup(func() { st.Close() })
32 if err := st.MigrateUp(); err != nil {
33 t.Fatal(err)
34 }
35 uid, err := st.CreateUser("alice", false)
36 if err != nil {
37 t.Fatal(err)
38 }
39 id, err := st.CreateRepo("user", uid, "app", "public")
40 if err != nil {
41 t.Fatal(err)
42 }
43 repo, err := st.RepoByID(id)
44 if err != nil {
45 t.Fatal(err)
46 }
47 root := t.TempDir()
48 f := &fixture{t: t, st: st, repo: repo, src: filepath.Join(root, "src"), bare: filepath.Join(root, "app.git")}
49 f.w = NewWith(st, func(owner, name string) string { return f.bare }, 0)
50 f.git(root, "init", "-q", "-b", repo.DefaultBranch, "src")
51 f.git(root, "init", "-q", "--bare", f.bare)
52 return f
53}
54
55func (f *fixture) git(dir string, args ...string) string {
56 f.t.Helper()
57 cmd := exec.Command("git", args...)
58 cmd.Dir = dir
59 cmd.Env = append(os.Environ(),
60 "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null",
61 "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.test",
62 "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.test")
63 out, err := cmd.CombinedOutput()
64 if err != nil {
65 f.t.Fatalf("git %v: %v\n%s", args, err, out)
66 }
67 return strings.TrimSpace(string(out))
68}
69
70// commit writes files (an empty content removes one) and pushes the
71// default branch to the bare repository.
72func (f *fixture) commit(files map[string]string) string {
73 f.t.Helper()
74 for name, content := range files {
75 p := filepath.Join(f.src, name)
76 if content == "" {
77 os.Remove(p)
78 continue
79 }
80 os.MkdirAll(filepath.Dir(p), 0o755)
81 if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
82 f.t.Fatal(err)
83 }
84 }
85 f.git(f.src, "add", "-A")
86 f.git(f.src, "commit", "-q", "--allow-empty", "-m", "c")
87 f.git(f.src, "push", "-q", "--force", f.bare, "HEAD:refs/heads/"+f.repo.DefaultBranch)
88 return f.git(f.src, "rev-parse", "HEAD")
89}
90
91func (f *fixture) sweep(force bool) store.SymbolIndex {
92 f.t.Helper()
93 if err := f.st.RequestSymbolIndex(f.repo.ID, force); err != nil {
94 f.t.Fatal(err)
95 }
96 f.w.Sweep(context.Background())
97 if reqs, _ := f.st.SymbolRequests(); len(reqs) != 0 {
98 f.t.Fatalf("request left after a sweep: %+v", reqs)
99 }
100 x, err := f.st.SymbolIndexFor(f.repo.ID)
101 if err != nil {
102 f.t.Fatal(err)
103 }
104 return x
105}
106
107func (f *fixture) names(x store.SymbolIndex) []string {
108 f.t.Helper()
109 rows, err := f.st.SearchSymbols(x.ID, "", "", 0, 0)
110 if err != nil {
111 f.t.Fatal(err)
112 }
113 var out []string
114 for _, r := range rows {
115 out = append(out, r.Path+":"+r.Name)
116 }
117 return out
118}
119
120func TestWorkerIndexesDefaultBranch(t *testing.T) {
121 f := newFixture(t)
122 head := f.commit(map[string]string{
123 "main.go": "package main\n\nfunc Hello() {}\n",
124 "vendor/dep/dep.go": "package dep\n\nfunc Vendored() {}\n",
125 "gen/zz_gen.go": "package gen\n\nfunc Generated() {}\n",
126 "big/big.go": "package big\n\nfunc Big() {}\n" + strings.Repeat("//\n", MaxFileBytes/3),
127 "docs/guide.md": "# Guide\n",
128 "assets/logo.png": "\x89PNG\r\n",
129 "node_modules/x.js": "function hidden() {}\n",
130 "web/app.js": "function shown() {}\n",
131 })
132 x := f.sweep(false)
133 if x.State != "ok" || x.Commit != head {
134 t.Fatalf("index = %+v, want ok at %s", x, head)
135 }
136 got := strings.Join(f.names(x), " ")
137 for _, want := range []string{"main.go:Hello", "docs/guide.md:Guide", "web/app.js:shown"} {
138 if !strings.Contains(got, want) {
139 t.Errorf("missing %s in %s", want, got)
140 }
141 }
142 for _, skip := range []string{"Vendored", "Generated", "Big", "hidden"} {
143 if strings.Contains(got, skip) {
144 t.Errorf("%s indexed: %s", skip, got)
145 }
146 }
147 if x.Files != 3 || x.Symbols != 3 {
148 t.Errorf("files %d symbols %d, want 3 and 3", x.Files, x.Symbols)
149 }
150}
151
152// A new commit whose tree is the indexed one is not indexed again; a
153// changed tree replaces the index, and force rebuilds an unchanged one.
154func TestWorkerKeysOnTree(t *testing.T) {
155 f := newFixture(t)
156 f.commit(map[string]string{"a.go": "package a\n\nfunc One() {}\n"})
157 first := f.sweep(false)
158
159 f.commit(nil) // same tree, new commit
160 same := f.sweep(false)
161 if same.ID != first.ID || same.Commit != first.Commit {
162 t.Fatalf("unchanged tree was reindexed: %+v then %+v", first, same)
163 }
164
165 f.commit(map[string]string{"a.go": "package a\n\nfunc Two() {}\n"})
166 changed := f.sweep(false)
167 if changed.ID == first.ID || changed.Tree == first.Tree {
168 t.Fatalf("changed tree kept the old index: %+v", changed)
169 }
170 if got := f.names(changed); len(got) != 1 || got[0] != "a.go:Two" {
171 t.Fatalf("symbols = %v, want only the new tree's", got)
172 }
173 var n int
174 f.st.DB.QueryRow("SELECT COUNT(*) FROM symbol_indexes WHERE repo_id = ?", f.repo.ID).Scan(&n)
175 if n != 1 {
176 t.Fatalf("%d indexes for one repository, want 1", n)
177 }
178
179 forced := f.sweep(true)
180 if forced.ID == changed.ID || forced.Tree != changed.Tree {
181 t.Fatalf("force did not rebuild: %+v then %+v", changed, forced)
182 }
183}
184
185func TestWorkerSymbolCap(t *testing.T) {
186 f := newFixture(t)
187 f.commit(map[string]string{"a.go": "package a\n\nfunc A() {}\nfunc B() {}\nfunc C() {}\n"})
188 f.w.MaxSymbols = 2
189 x := f.sweep(false)
190 if x.State != "partial" || x.Symbols != 2 || !strings.Contains(x.Note, "2 symbols") {
191 t.Fatalf("index = %+v, want partial with 2 symbols", x)
192 }
193}
194
195func TestWorkerTimeBound(t *testing.T) {
196 f := newFixture(t)
197 f.commit(map[string]string{"a.go": "package a\n\nfunc A() {}\n"})
198 f.w.MaxTime = time.Nanosecond
199 x := f.sweep(false)
200 if x.State != "partial" || !strings.Contains(x.Note, "stopped after") {
201 t.Fatalf("index = %+v, want partial on the time bound", x)
202 }
203}
204
205// breakBlob deletes the loose object of path at the default branch, so
206// the tree cannot be read.
207func (f *fixture) breakBlob(path string) {
208 f.t.Helper()
209 blob := f.git(f.bare, "rev-parse", "refs/heads/"+f.repo.DefaultBranch+":"+path)
210 if err := os.Remove(filepath.Join(f.bare, "objects", blob[:2], blob[2:])); err != nil {
211 f.t.Fatal(err)
212 }
213}
214
215// A tree that cannot be read is recorded as a failure beside the current
216// index, which stays current. The request is kept for one retry after the
217// backoff rather than tried again at once; a new request for the same
218// tree inside the backoff is not retried either.
219func TestWorkerFailureKeepsCurrentIndex(t *testing.T) {
220 f := newFixture(t)
221 f.commit(map[string]string{"a.go": "package a\n\nfunc A() {}\n"})
222 good := f.sweep(false)
223
224 f.commit(map[string]string{"b.go": "package a\n\nfunc B() {}\n"})
225 f.breakBlob("b.go")
226 f.st.RequestSymbolIndex(f.repo.ID, false)
227 f.w.Sweep(context.Background())
228 cur, err := f.st.SymbolIndexFor(f.repo.ID)
229 if err != nil || cur.ID != good.ID {
230 t.Fatalf("current index after a failure = %+v, %v; want %d", cur, err, good.ID)
231 }
232 fail, err := f.st.SymbolFailureFor(f.repo.ID)
233 if err != nil || !strings.Contains(fail.Note, "b.go") {
234 t.Fatalf("failure = %+v, %v", fail, err)
235 }
236 if reqs, _ := f.st.SymbolRequests(); len(reqs) != 0 {
237 t.Fatalf("a failed request is due again at once: %+v", reqs)
238 }
239 var attempts int
240 f.st.DB.QueryRow("SELECT attempts FROM symbol_requests WHERE repo_id = ?", f.repo.ID).Scan(&attempts)
241 if attempts != 1 {
242 t.Fatalf("attempts = %d, want the request kept for one retry", attempts)
243 }
244
245 // Asked again inside the backoff: the failed tree is left alone.
246 f.st.RequestSymbolIndex(f.repo.ID, false)
247 failed, err := f.w.Index(context.Background(), f.repo.ID, false)
248 if failed || err != nil {
249 t.Fatalf("a recently failed tree was retried: %v, %v", failed, err)
250 }
251
252 // After the backoff it is tried again, and still fails.
253 f.w.Backoff = 0
254 if failed, _ := f.w.Index(context.Background(), f.repo.ID, false); !failed {
255 t.Fatal("a failed tree was not retried after the backoff")
256 }
257
258 // A second failure of a retry is the end of it.
259 f.st.DB.Exec("DELETE FROM symbol_requests")
260 f.st.DB.Exec("INSERT INTO symbol_requests (repo_id, attempts) VALUES (?, 1)", f.repo.ID)
261 f.w.Sweep(context.Background())
262 var n int
263 f.st.DB.QueryRow("SELECT COUNT(*) FROM symbol_requests").Scan(&n)
264 if n != 0 {
265 t.Fatal("a failed retry was kept for another")
266 }
267
268 // A push that changes the tree builds, and clears the failure.
269 f.git(f.src, "rm", "-q", "b.go")
270 f.commit(map[string]string{"c.go": "package a\n\nfunc C() {}\n"})
271 next := f.sweep(false)
272 if next.ID == good.ID {
273 t.Fatal("a new tree after a failure was not indexed")
274 }
275 if _, err := f.st.SymbolFailureFor(f.repo.ID); err != store.ErrNotFound {
276 t.Fatalf("failure kept after a good index: %v", err)
277 }
278}
279
280// A new index is written in chunks no read sees: throughout the build the
281// old index is current, and the flip replaces it whole.
282func TestWorkerBuildIsInvisibleUntilPublished(t *testing.T) {
283 f := newFixture(t)
284 f.commit(map[string]string{"a.go": "package a\n\nfunc Old() {}\n"})
285 old := f.sweep(false)
286
287 f.commit(map[string]string{"a.go": "package a\n\nfunc New1() {}\nfunc New2() {}\nfunc New3() {}\n"})
288 f.w.ChunkRows = 1
289 chunks := 0
290 f.w.chunkHook = func(building int64) {
291 chunks++
292 cur, err := f.st.SymbolIndexFor(f.repo.ID)
293 if err != nil || cur.ID != old.ID {
294 t.Errorf("mid-build current index = %+v, %v; want the old one", cur, err)
295 }
296 if got := f.names(cur); len(got) != 1 || got[0] != "a.go:Old" {
297 t.Errorf("mid-build reads %v", got)
298 }
299 if b, _ := f.st.SymbolIndexByID(building); b.State != "building" {
300 t.Errorf("index being built is %q", b.State)
301 }
302 }
303 next := f.sweep(false)
304 if chunks != 3 {
305 t.Fatalf("%d chunks, want one per row", chunks)
306 }
307 if got := f.names(next); len(got) != 3 {
308 t.Fatalf("after the flip: %v", got)
309 }
310 var indexes, rows int
311 f.st.DB.QueryRow("SELECT COUNT(*) FROM symbol_indexes").Scan(&indexes)
312 f.st.DB.QueryRow("SELECT COUNT(*) FROM symbols").Scan(&rows)
313 if indexes != 1 || rows != 3 {
314 t.Fatalf("%d indexes and %d rows left, want the new index alone", indexes, rows)
315 }
316}
317
318// A run that stopped mid-build leaves the old index current, and the next
319// run deletes what it wrote.
320func TestWorkerCleansUpInterruptedBuild(t *testing.T) {
321 f := newFixture(t)
322 f.commit(map[string]string{"a.go": "package a\n\nfunc Old() {}\n"})
323 old := f.sweep(false)
324
325 // What a crash between chunks leaves.
326 orphan, err := f.st.BeginSymbolIndex(f.repo.ID, "c", "t")
327 if err != nil {
328 t.Fatal(err)
329 }
330 if err := f.st.AddSymbols(orphan, []store.SymbolRow{{Name: "Half", Key: "Half", Kind: "function", Path: "x.go", Line: 1}}); err != nil {
331 t.Fatal(err)
332 }
333 if cur, _ := f.st.SymbolIndexFor(f.repo.ID); cur.ID != old.ID {
334 t.Fatalf("an unfinished build became current: %+v", cur)
335 }
336
337 f.commit(nil) // same tree: nothing to build, but the leftovers go
338 f.sweep(false)
339 if _, err := f.st.SymbolIndexByID(orphan); err != store.ErrNotFound {
340 t.Fatalf("interrupted build still there: %v", err)
341 }
342 var rows int
343 f.st.DB.QueryRow("SELECT COUNT(*) FROM symbols WHERE index_id = ?", orphan).Scan(&rows)
344 if rows != 0 {
345 t.Fatalf("%d rows of the interrupted build left", rows)
346 }
347}
348
349// A hostile tree: names past the length cap are dropped, and long
350// headings stop at the byte budget with a partial index saying so.
351func TestWorkerHostileNames(t *testing.T) {
352 f := newFixture(t)
353 files := map[string]string{}
354 huge := strings.Repeat("x", MaxNameBytes+1)
355 for i := 0; i < 20; i++ {
356 files[fmt.Sprintf("huge%02d.go", i)] = "package a\n\nfunc " + huge + "() {}\nvar " + huge + " int\n"
357 }
358 var md strings.Builder
359 for i := 0; i < 500; i++ {
360 fmt.Fprintf(&md, "# %03d %s\n", i, strings.Repeat("h", 200))
361 }
362 files["notes.md"] = md.String()
363 f.commit(files)
364 f.w.MaxBytes = 20_000
365 x := f.sweep(false)
366 if x.State != "partial" || !strings.Contains(x.Note, "20000 bytes") {
367 t.Fatalf("index = %+v, want partial at the byte budget", x)
368 }
369 var longest, total int
370 f.st.DB.QueryRow("SELECT COALESCE(MAX(length(name)), 0), COALESCE(SUM(length(name) + length(key) + length(path)), 0) FROM symbols").
371 Scan(&longest, &total)
372 if longest > MaxNameBytes {
373 t.Errorf("a %d-byte name was kept", longest)
374 }
375 if total > 20_000 || x.Symbols == 0 {
376 t.Errorf("%d symbols holding %d bytes, want some within the budget", x.Symbols, total)
377 }
378}
379
380func TestWorkerIgnoresEmptyRepository(t *testing.T) {
381 f := newFixture(t)
382 f.st.RequestSymbolIndex(f.repo.ID, false)
383 f.w.Sweep(context.Background())
384 if _, err := f.st.SymbolIndexFor(f.repo.ID); err != store.ErrNotFound {
385 t.Fatalf("an empty repository has an index: %v", err)
386 }
387 if reqs, _ := f.st.SymbolRequests(); len(reqs) != 0 {
388 t.Fatalf("request left: %+v", reqs)
389 }
390}