internal/symbols/worker_test.go

  1package symbols
  2
  3import (
  4	"context"
  5	"fmt"
  6	"os"
  7	"os/exec"
  8	"path/filepath"
  9	"strings"
 10	"testing"
 11	"time"
 12
 13	"gitbay.org/gitbay/internal/store"
 14)
 15
 16type fixture struct {
 17	t    *testing.T
 18	st   *store.Store
 19	repo store.Repo
 20	src  string
 21	bare string
 22	w    *Worker
 23}
 24
 25func newFixture(t *testing.T) *fixture {
 26	t.Helper()
 27	st, err := store.Open(":memory:")
 28	if err != nil {
 29		t.Fatal(err)
 30	}
 31	t.Cleanup(func() { st.Close() })
 32	if err := st.MigrateUp(); err != nil {
 33		t.Fatal(err)
 34	}
 35	uid, err := st.CreateUser("alice", false)
 36	if err != nil {
 37		t.Fatal(err)
 38	}
 39	id, err := st.CreateRepo("user", uid, "app", "public")
 40	if err != nil {
 41		t.Fatal(err)
 42	}
 43	repo, err := st.RepoByID(id)
 44	if err != nil {
 45		t.Fatal(err)
 46	}
 47	root := t.TempDir()
 48	f := &fixture{t: t, st: st, repo: repo, src: filepath.Join(root, "src"), bare: filepath.Join(root, "app.git")}
 49	f.w = NewWith(st, func(owner, name string) string { return f.bare }, 0)
 50	f.git(root, "init", "-q", "-b", repo.DefaultBranch, "src")
 51	f.git(root, "init", "-q", "--bare", f.bare)
 52	return f
 53}
 54
 55func (f *fixture) git(dir string, args ...string) string {
 56	f.t.Helper()
 57	cmd := exec.Command("git", args...)
 58	cmd.Dir = dir
 59	cmd.Env = append(os.Environ(),
 60		"GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null",
 61		"GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.test",
 62		"GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.test")
 63	out, err := cmd.CombinedOutput()
 64	if err != nil {
 65		f.t.Fatalf("git %v: %v\n%s", args, err, out)
 66	}
 67	return strings.TrimSpace(string(out))
 68}
 69
 70// commit writes files (an empty content removes one) and pushes the
 71// default branch to the bare repository.
 72func (f *fixture) commit(files map[string]string) string {
 73	f.t.Helper()
 74	for name, content := range files {
 75		p := filepath.Join(f.src, name)
 76		if content == "" {
 77			os.Remove(p)
 78			continue
 79		}
 80		os.MkdirAll(filepath.Dir(p), 0o755)
 81		if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
 82			f.t.Fatal(err)
 83		}
 84	}
 85	f.git(f.src, "add", "-A")
 86	f.git(f.src, "commit", "-q", "--allow-empty", "-m", "c")
 87	f.git(f.src, "push", "-q", "--force", f.bare, "HEAD:refs/heads/"+f.repo.DefaultBranch)
 88	return f.git(f.src, "rev-parse", "HEAD")
 89}
 90
 91func (f *fixture) sweep(force bool) store.SymbolIndex {
 92	f.t.Helper()
 93	if err := f.st.RequestSymbolIndex(f.repo.ID, force); err != nil {
 94		f.t.Fatal(err)
 95	}
 96	f.w.Sweep(context.Background())
 97	if reqs, _ := f.st.SymbolRequests(); len(reqs) != 0 {
 98		f.t.Fatalf("request left after a sweep: %+v", reqs)
 99	}
100	x, err := f.st.SymbolIndexFor(f.repo.ID)
101	if err != nil {
102		f.t.Fatal(err)
103	}
104	return x
105}
106
107func (f *fixture) names(x store.SymbolIndex) []string {
108	f.t.Helper()
109	rows, err := f.st.SearchSymbols(x.ID, "", "", 0, 0)
110	if err != nil {
111		f.t.Fatal(err)
112	}
113	var out []string
114	for _, r := range rows {
115		out = append(out, r.Path+":"+r.Name)
116	}
117	return out
118}
119
120func TestWorkerIndexesDefaultBranch(t *testing.T) {
121	f := newFixture(t)
122	head := f.commit(map[string]string{
123		"main.go":           "package main\n\nfunc Hello() {}\n",
124		"vendor/dep/dep.go": "package dep\n\nfunc Vendored() {}\n",
125		"gen/zz_gen.go":     "package gen\n\nfunc Generated() {}\n",
126		"big/big.go":        "package big\n\nfunc Big() {}\n" + strings.Repeat("//\n", MaxFileBytes/3),
127		"docs/guide.md":     "# Guide\n",
128		"assets/logo.png":   "\x89PNG\r\n",
129		"node_modules/x.js": "function hidden() {}\n",
130		"web/app.js":        "function shown() {}\n",
131	})
132	x := f.sweep(false)
133	if x.State != "ok" || x.Commit != head {
134		t.Fatalf("index = %+v, want ok at %s", x, head)
135	}
136	got := strings.Join(f.names(x), " ")
137	for _, want := range []string{"main.go:Hello", "docs/guide.md:Guide", "web/app.js:shown"} {
138		if !strings.Contains(got, want) {
139			t.Errorf("missing %s in %s", want, got)
140		}
141	}
142	for _, skip := range []string{"Vendored", "Generated", "Big", "hidden"} {
143		if strings.Contains(got, skip) {
144			t.Errorf("%s indexed: %s", skip, got)
145		}
146	}
147	if x.Files != 3 || x.Symbols != 3 {
148		t.Errorf("files %d symbols %d, want 3 and 3", x.Files, x.Symbols)
149	}
150}
151
152// A new commit whose tree is the indexed one is not indexed again; a
153// changed tree replaces the index, and force rebuilds an unchanged one.
154func TestWorkerKeysOnTree(t *testing.T) {
155	f := newFixture(t)
156	f.commit(map[string]string{"a.go": "package a\n\nfunc One() {}\n"})
157	first := f.sweep(false)
158
159	f.commit(nil) // same tree, new commit
160	same := f.sweep(false)
161	if same.ID != first.ID || same.Commit != first.Commit {
162		t.Fatalf("unchanged tree was reindexed: %+v then %+v", first, same)
163	}
164
165	f.commit(map[string]string{"a.go": "package a\n\nfunc Two() {}\n"})
166	changed := f.sweep(false)
167	if changed.ID == first.ID || changed.Tree == first.Tree {
168		t.Fatalf("changed tree kept the old index: %+v", changed)
169	}
170	if got := f.names(changed); len(got) != 1 || got[0] != "a.go:Two" {
171		t.Fatalf("symbols = %v, want only the new tree's", got)
172	}
173	var n int
174	f.st.DB.QueryRow("SELECT COUNT(*) FROM symbol_indexes WHERE repo_id = ?", f.repo.ID).Scan(&n)
175	if n != 1 {
176		t.Fatalf("%d indexes for one repository, want 1", n)
177	}
178
179	forced := f.sweep(true)
180	if forced.ID == changed.ID || forced.Tree != changed.Tree {
181		t.Fatalf("force did not rebuild: %+v then %+v", changed, forced)
182	}
183}
184
185func TestWorkerSymbolCap(t *testing.T) {
186	f := newFixture(t)
187	f.commit(map[string]string{"a.go": "package a\n\nfunc A() {}\nfunc B() {}\nfunc C() {}\n"})
188	f.w.MaxSymbols = 2
189	x := f.sweep(false)
190	if x.State != "partial" || x.Symbols != 2 || !strings.Contains(x.Note, "2 symbols") {
191		t.Fatalf("index = %+v, want partial with 2 symbols", x)
192	}
193}
194
195func TestWorkerTimeBound(t *testing.T) {
196	f := newFixture(t)
197	f.commit(map[string]string{"a.go": "package a\n\nfunc A() {}\n"})
198	f.w.MaxTime = time.Nanosecond
199	x := f.sweep(false)
200	if x.State != "partial" || !strings.Contains(x.Note, "stopped after") {
201		t.Fatalf("index = %+v, want partial on the time bound", x)
202	}
203}
204
205// breakBlob deletes the loose object of path at the default branch, so
206// the tree cannot be read.
207func (f *fixture) breakBlob(path string) {
208	f.t.Helper()
209	blob := f.git(f.bare, "rev-parse", "refs/heads/"+f.repo.DefaultBranch+":"+path)
210	if err := os.Remove(filepath.Join(f.bare, "objects", blob[:2], blob[2:])); err != nil {
211		f.t.Fatal(err)
212	}
213}
214
215// A tree that cannot be read is recorded as a failure beside the current
216// index, which stays current. The request is kept for one retry after the
217// backoff rather than tried again at once; a new request for the same
218// tree inside the backoff is not retried either.
219func TestWorkerFailureKeepsCurrentIndex(t *testing.T) {
220	f := newFixture(t)
221	f.commit(map[string]string{"a.go": "package a\n\nfunc A() {}\n"})
222	good := f.sweep(false)
223
224	f.commit(map[string]string{"b.go": "package a\n\nfunc B() {}\n"})
225	f.breakBlob("b.go")
226	f.st.RequestSymbolIndex(f.repo.ID, false)
227	f.w.Sweep(context.Background())
228	cur, err := f.st.SymbolIndexFor(f.repo.ID)
229	if err != nil || cur.ID != good.ID {
230		t.Fatalf("current index after a failure = %+v, %v; want %d", cur, err, good.ID)
231	}
232	fail, err := f.st.SymbolFailureFor(f.repo.ID)
233	if err != nil || !strings.Contains(fail.Note, "b.go") {
234		t.Fatalf("failure = %+v, %v", fail, err)
235	}
236	if reqs, _ := f.st.SymbolRequests(); len(reqs) != 0 {
237		t.Fatalf("a failed request is due again at once: %+v", reqs)
238	}
239	var attempts int
240	f.st.DB.QueryRow("SELECT attempts FROM symbol_requests WHERE repo_id = ?", f.repo.ID).Scan(&attempts)
241	if attempts != 1 {
242		t.Fatalf("attempts = %d, want the request kept for one retry", attempts)
243	}
244
245	// Asked again inside the backoff: the failed tree is left alone.
246	f.st.RequestSymbolIndex(f.repo.ID, false)
247	failed, err := f.w.Index(context.Background(), f.repo.ID, false)
248	if failed || err != nil {
249		t.Fatalf("a recently failed tree was retried: %v, %v", failed, err)
250	}
251
252	// After the backoff it is tried again, and still fails.
253	f.w.Backoff = 0
254	if failed, _ := f.w.Index(context.Background(), f.repo.ID, false); !failed {
255		t.Fatal("a failed tree was not retried after the backoff")
256	}
257
258	// A second failure of a retry is the end of it.
259	f.st.DB.Exec("DELETE FROM symbol_requests")
260	f.st.DB.Exec("INSERT INTO symbol_requests (repo_id, attempts) VALUES (?, 1)", f.repo.ID)
261	f.w.Sweep(context.Background())
262	var n int
263	f.st.DB.QueryRow("SELECT COUNT(*) FROM symbol_requests").Scan(&n)
264	if n != 0 {
265		t.Fatal("a failed retry was kept for another")
266	}
267
268	// A push that changes the tree builds, and clears the failure.
269	f.git(f.src, "rm", "-q", "b.go")
270	f.commit(map[string]string{"c.go": "package a\n\nfunc C() {}\n"})
271	next := f.sweep(false)
272	if next.ID == good.ID {
273		t.Fatal("a new tree after a failure was not indexed")
274	}
275	if _, err := f.st.SymbolFailureFor(f.repo.ID); err != store.ErrNotFound {
276		t.Fatalf("failure kept after a good index: %v", err)
277	}
278}
279
280// A new index is written in chunks no read sees: throughout the build the
281// old index is current, and the flip replaces it whole.
282func TestWorkerBuildIsInvisibleUntilPublished(t *testing.T) {
283	f := newFixture(t)
284	f.commit(map[string]string{"a.go": "package a\n\nfunc Old() {}\n"})
285	old := f.sweep(false)
286
287	f.commit(map[string]string{"a.go": "package a\n\nfunc New1() {}\nfunc New2() {}\nfunc New3() {}\n"})
288	f.w.ChunkRows = 1
289	chunks := 0
290	f.w.chunkHook = func(building int64) {
291		chunks++
292		cur, err := f.st.SymbolIndexFor(f.repo.ID)
293		if err != nil || cur.ID != old.ID {
294			t.Errorf("mid-build current index = %+v, %v; want the old one", cur, err)
295		}
296		if got := f.names(cur); len(got) != 1 || got[0] != "a.go:Old" {
297			t.Errorf("mid-build reads %v", got)
298		}
299		if b, _ := f.st.SymbolIndexByID(building); b.State != "building" {
300			t.Errorf("index being built is %q", b.State)
301		}
302	}
303	next := f.sweep(false)
304	if chunks != 3 {
305		t.Fatalf("%d chunks, want one per row", chunks)
306	}
307	if got := f.names(next); len(got) != 3 {
308		t.Fatalf("after the flip: %v", got)
309	}
310	var indexes, rows int
311	f.st.DB.QueryRow("SELECT COUNT(*) FROM symbol_indexes").Scan(&indexes)
312	f.st.DB.QueryRow("SELECT COUNT(*) FROM symbols").Scan(&rows)
313	if indexes != 1 || rows != 3 {
314		t.Fatalf("%d indexes and %d rows left, want the new index alone", indexes, rows)
315	}
316}
317
318// A run that stopped mid-build leaves the old index current, and the next
319// run deletes what it wrote.
320func TestWorkerCleansUpInterruptedBuild(t *testing.T) {
321	f := newFixture(t)
322	f.commit(map[string]string{"a.go": "package a\n\nfunc Old() {}\n"})
323	old := f.sweep(false)
324
325	// What a crash between chunks leaves.
326	orphan, err := f.st.BeginSymbolIndex(f.repo.ID, "c", "t")
327	if err != nil {
328		t.Fatal(err)
329	}
330	if err := f.st.AddSymbols(orphan, []store.SymbolRow{{Name: "Half", Key: "Half", Kind: "function", Path: "x.go", Line: 1}}); err != nil {
331		t.Fatal(err)
332	}
333	if cur, _ := f.st.SymbolIndexFor(f.repo.ID); cur.ID != old.ID {
334		t.Fatalf("an unfinished build became current: %+v", cur)
335	}
336
337	f.commit(nil) // same tree: nothing to build, but the leftovers go
338	f.sweep(false)
339	if _, err := f.st.SymbolIndexByID(orphan); err != store.ErrNotFound {
340		t.Fatalf("interrupted build still there: %v", err)
341	}
342	var rows int
343	f.st.DB.QueryRow("SELECT COUNT(*) FROM symbols WHERE index_id = ?", orphan).Scan(&rows)
344	if rows != 0 {
345		t.Fatalf("%d rows of the interrupted build left", rows)
346	}
347}
348
349// A hostile tree: names past the length cap are dropped, and long
350// headings stop at the byte budget with a partial index saying so.
351func TestWorkerHostileNames(t *testing.T) {
352	f := newFixture(t)
353	files := map[string]string{}
354	huge := strings.Repeat("x", MaxNameBytes+1)
355	for i := 0; i < 20; i++ {
356		files[fmt.Sprintf("huge%02d.go", i)] = "package a\n\nfunc " + huge + "() {}\nvar " + huge + " int\n"
357	}
358	var md strings.Builder
359	for i := 0; i < 500; i++ {
360		fmt.Fprintf(&md, "# %03d %s\n", i, strings.Repeat("h", 200))
361	}
362	files["notes.md"] = md.String()
363	f.commit(files)
364	f.w.MaxBytes = 20_000
365	x := f.sweep(false)
366	if x.State != "partial" || !strings.Contains(x.Note, "20000 bytes") {
367		t.Fatalf("index = %+v, want partial at the byte budget", x)
368	}
369	var longest, total int
370	f.st.DB.QueryRow("SELECT COALESCE(MAX(length(name)), 0), COALESCE(SUM(length(name) + length(key) + length(path)), 0) FROM symbols").
371		Scan(&longest, &total)
372	if longest > MaxNameBytes {
373		t.Errorf("a %d-byte name was kept", longest)
374	}
375	if total > 20_000 || x.Symbols == 0 {
376		t.Errorf("%d symbols holding %d bytes, want some within the budget", x.Symbols, total)
377	}
378}
379
380func TestWorkerIgnoresEmptyRepository(t *testing.T) {
381	f := newFixture(t)
382	f.st.RequestSymbolIndex(f.repo.ID, false)
383	f.w.Sweep(context.Background())
384	if _, err := f.st.SymbolIndexFor(f.repo.ID); err != store.ErrNotFound {
385		t.Fatalf("an empty repository has an index: %v", err)
386	}
387	if reqs, _ := f.st.SymbolRequests(); len(reqs) != 0 {
388		t.Fatalf("request left: %+v", reqs)
389	}
390}