Commit `f686b22ad7`

f686b22ad7a1bca7b15aee6a44af67e7b8efade1

parent: 81f47453f4

Verified · cmc ci/build: success ci/test: success ci/vuln: success

cmc <hello@cleberg.net> · 2026-09-04 14:51 UTC

store: the dashboard walks an index instead of sorting the table

Every dashboard list scanned its table and sorted the result to take
fifty rows. On 20k issues and 20k merge requests:

  DashboardIssues  11.8ms -> 0.8ms
  DashboardMRs     12.1ms -> 0.8ms
  ReviewQueue      15.5ms -> 0.3ms
  AssignedIssues   10.6ms -> 0.04ms

Reachability is a set of correlated subqueries and cannot be indexed, so
the index does not narrow the filter; it supplies the ORDER BY, and the
walk stops at LIMIT.

The indexes the audit named — (author_id, updated_at) on both tables —
are not these. Nothing can use them: the filter is `author_id = ? OR
reachable`, and an OR across a column and a correlated subquery scans
either way. Rewriting it as a UNION so the author branch could use one
measured slower than leaving it alone, 11.9ms to 15.3ms.

(state, updated_at) is also wrong, and worse than nothing on the merge
request lists: they match state with IN, which turns one ordered walk
into two that must be merged, 12.1ms to 19.6ms.

AssignedIssues drives from issue_assignees rather than testing EXISTS
against every issue; issue_assignees' primary key leads with issue_id,
so there was no way in by user.

TestDashboardQueriesUseIndexes asserts each plan still walks its index,
since losing one costs an order of magnitude and shows no other symptom.

Closes #137

Layout: unified · split

internal/store/dashboard.go +60 −47

		@@ -45,33 +45,39 @@ func (s *Store) dashboardQuery(q string, userID int64) ([]DashboardItem, error)
45	45	return out, rows.Err()
46	46	}
47	47
	48	// The dashboard's four list queries are named so the plan test can assert
	49	// each still walks the 0035 index that supplies its ORDER BY.
	50	const dashboardMRsQuery = `
	51	SELECT COALESCE(u.username, o.name) \|\| '/' \|\| r.name,
	52	x.number, x.title, au.username, x.state, x.updated_at
	53	FROM merge_requests x
	54	JOIN repos r ON r.id = x.repo_id
	55	LEFT JOIN users u ON r.owner_kind = 'user' AND u.id = r.owner_id
	56	LEFT JOIN orgs o ON r.owner_kind = 'org' AND o.id = r.owner_id
	57	JOIN users au ON au.id = x.author_id
	58	WHERE x.state IN ('open', 'source_gone') AND ` + involvedCond + `
	59	ORDER BY x.updated_at DESC LIMIT 50`
	60
48	61	// DashboardMRs returns open merge requests involving the user: on their
49	62	// repositories (owned, granted, org) or authored by them anywhere.
50	63	func (s *Store) DashboardMRs(userID int64) ([]DashboardItem, error) {
51		return s.dashboardQuery(`
52		SELECT COALESCE(u.username, o.name) \|\| '/' \|\| r.name,
53		x.number, x.title, au.username, x.state, x.updated_at
54		FROM merge_requests x
55		JOIN repos r ON r.id = x.repo_id
56		LEFT JOIN users u ON r.owner_kind = 'user' AND u.id = r.owner_id
57		LEFT JOIN orgs o ON r.owner_kind = 'org' AND o.id = r.owner_id
58		JOIN users au ON au.id = x.author_id
59		WHERE x.state IN ('open', 'source_gone') AND `+involvedCond+`
60		ORDER BY x.updated_at DESC LIMIT 50`, userID)
	64	return s.dashboardQuery(dashboardMRsQuery, userID)
61	65	}
62	66
63	67	// DashboardIssues is the issue counterpart of DashboardMRs.
	68	const dashboardIssuesQuery = `
	69	SELECT COALESCE(u.username, o.name) \|\| '/' \|\| r.name,
	70	x.number, x.title, au.username, x.state, x.updated_at
	71	FROM issues x
	72	JOIN repos r ON r.id = x.repo_id
	73	LEFT JOIN users u ON r.owner_kind = 'user' AND u.id = r.owner_id
	74	LEFT JOIN orgs o ON r.owner_kind = 'org' AND o.id = r.owner_id
	75	JOIN users au ON au.id = x.author_id
	76	WHERE x.state = 'open' AND ` + involvedCond + `
	77	ORDER BY x.updated_at DESC LIMIT 50`
	78
64	79	func (s *Store) DashboardIssues(userID int64) ([]DashboardItem, error) {
65		return s.dashboardQuery(`
66		SELECT COALESCE(u.username, o.name) \|\| '/' \|\| r.name,
67		x.number, x.title, au.username, x.state, x.updated_at
68		FROM issues x
69		JOIN repos r ON r.id = x.repo_id
70		LEFT JOIN users u ON r.owner_kind = 'user' AND u.id = r.owner_id
71		LEFT JOIN orgs o ON r.owner_kind = 'org' AND o.id = r.owner_id
72		JOIN users au ON au.id = x.author_id
73		WHERE x.state = 'open' AND `+involvedCond+`
74		ORDER BY x.updated_at DESC LIMIT 50`, userID)
	80	return s.dashboardQuery(dashboardIssuesQuery, userID)
75	81	}
76	82
77	83	func (s *Store) PinRepo(userID, repoID int64) error {
		@@ -124,22 +130,24 @@ func (s *Store) PinnedRepos(userID int64) ([]Repo, error) {
124	130	// ReviewQueue returns open merge requests the user is involved in, has not
125	131	// authored, and has not reviewed at the current head — what the rail shows
126	132	// as waiting on them. Ordered most recently touched first.
	133	const reviewQueueQuery = `
	134	SELECT COALESCE(u.username, o.name) \|\| '/' \|\| r.name,
	135	x.number, x.title, au.username, x.state, x.updated_at
	136	FROM merge_requests x
	137	JOIN repos r ON r.id = x.repo_id
	138	LEFT JOIN users u ON r.owner_kind = 'user' AND u.id = r.owner_id
	139	LEFT JOIN orgs o ON r.owner_kind = 'org' AND o.id = r.owner_id
	140	JOIN users au ON au.id = x.author_id
	141	WHERE x.state IN ('open', 'source_gone')
	142	AND x.author_id <> ?1
	143	AND NOT EXISTS (SELECT 1 FROM mr_reviews rv
	144	WHERE rv.mr_id = x.id AND rv.reviewer_id = ?1
	145	AND rv.head_sha = x.head_sha)
	146	AND ` + involvedCond + `
	147	ORDER BY x.updated_at DESC LIMIT 8`
	148
127	149	func (s *Store) ReviewQueue(userID int64) ([]DashboardItem, error) {
128		return s.dashboardQuery(`
129		SELECT COALESCE(u.username, o.name) \|\| '/' \|\| r.name,
130		x.number, x.title, au.username, x.state, x.updated_at
131		FROM merge_requests x
132		JOIN repos r ON r.id = x.repo_id
133		LEFT JOIN users u ON r.owner_kind = 'user' AND u.id = r.owner_id
134		LEFT JOIN orgs o ON r.owner_kind = 'org' AND o.id = r.owner_id
135		JOIN users au ON au.id = x.author_id
136		WHERE x.state IN ('open', 'source_gone')
137		AND x.author_id <> ?1
138		AND NOT EXISTS (SELECT 1 FROM mr_reviews rv
139		WHERE rv.mr_id = x.id AND rv.reviewer_id = ?1
140		AND rv.head_sha = x.head_sha)
141		AND `+involvedCond+`
142		ORDER BY x.updated_at DESC LIMIT 8`, userID)
	150	return s.dashboardQuery(reviewQueueQuery, userID)
143	151	}
144	152
145	153	// OpenCounts returns the repo's open issue and open merge request counts,
		@@ -155,19 +163,24 @@ func (s *Store) OpenCounts(repoID int64) (issues, mrs int) {
155	163	// AssignedIssues returns open issues assigned to the user, wherever they
156	164	// live. Assignment is a direct request for someone's attention, so it is
157	165	// not narrowed by the involvement rule the other lists use.
	166	//
	167	// It drives from issue_assignees rather than testing EXISTS against every
	168	// issue: the assignee rows for one user are a handful, the issues table
	169	// is the whole instance.
	170	const assignedIssuesQuery = `
	171	SELECT COALESCE(u.username, o.name) \|\| '/' \|\| r.name,
	172	x.number, x.title, au.username, x.state, x.updated_at
	173	FROM issue_assignees ia
	174	JOIN issues x ON x.id = ia.issue_id
	175	JOIN repos r ON r.id = x.repo_id
	176	LEFT JOIN users u ON r.owner_kind = 'user' AND u.id = r.owner_id
	177	LEFT JOIN orgs o ON r.owner_kind = 'org' AND o.id = r.owner_id
	178	JOIN users au ON au.id = x.author_id
	179	WHERE ia.user_id = ?1 AND x.state = 'open'
	180	ORDER BY x.updated_at DESC LIMIT 20`
	181
158	182	func (s *Store) AssignedIssues(userID int64) ([]DashboardItem, error) {
159		return s.dashboardQuery(`
160		SELECT COALESCE(u.username, o.name) \|\| '/' \|\| r.name,
161		x.number, x.title, au.username, x.state, x.updated_at
162		FROM issues x
163		JOIN repos r ON r.id = x.repo_id
164		LEFT JOIN users u ON r.owner_kind = 'user' AND u.id = r.owner_id
165		LEFT JOIN orgs o ON r.owner_kind = 'org' AND o.id = r.owner_id
166		JOIN users au ON au.id = x.author_id
167		WHERE x.state = 'open'
168		AND EXISTS (SELECT 1 FROM issue_assignees ia
169		WHERE ia.issue_id = x.id AND ia.user_id = ?1)
170		ORDER BY x.updated_at DESC LIMIT 20`, userID)
	183	return s.dashboardQuery(assignedIssuesQuery, userID)
171	184	}
172	185
173	186	// DashboardBuild is one build row on the dashboard, with its repo resolved.

internal/store/dashboardplan_test.go added +66

		@@ -0,0 +1,66 @@
	1	package store
	2
	3	import (
	4	"strings"
	5	"testing"
	6	)
	7
	8	// queryPlan returns EXPLAIN QUERY PLAN for q as one string.
	9	func queryPlan(t testing.T, s Store, q string, args ...any) string {
	10	t.Helper()
	11	rows, err := s.DB.Query("EXPLAIN QUERY PLAN "+q, args...)
	12	if err != nil {
	13	t.Fatalf("explain: %v", err)
	14	}
	15	defer rows.Close()
	16	var b strings.Builder
	17	for rows.Next() {
	18	var id, parent, notUsed int
	19	var detail string
	20	if err := rows.Scan(&id, &parent, &notUsed, &detail); err != nil {
	21	t.Fatal(err)
	22	}
	23	b.WriteString(detail)
	24	b.WriteString("\n")
	25	}
	26	return b.String()
	27	}
	28
	29	// TestDashboardQueriesUseIndexes guards the plans the 0035 indexes exist
	30	// for. Reachability is correlated subqueries and cannot be indexed, so
	31	// what these queries buy from an index is the ORDER BY: the walk stops at
	32	// LIMIT rather than sorting the table. A rewrite that reintroduces a sort
	33	// puts the cost back — 0.8ms to 11.8ms on 20k issues — with no other
	34	// symptom, which is what this asserts against (#137).
	35	func TestDashboardQueriesUseIndexes(t *testing.T) {
	36	s := open(t)
	37	if err := s.MigrateUp(); err != nil {
	38	t.Fatal(err)
	39	}
	40	// The planner picks against an empty table the same way it does
	41	// against a full one here: these plans are driven by the ORDER BY and
	42	// the index's presence, not by row counts.
	43	cases := []struct {
	44	name string
	45	plan string
	46	want string
	47	// ordered marks the queries whose index supplies the ORDER BY, so
	48	// a sort in the plan means the walk is back to reading every row.
	49	// AssignedIssues is not one: it drives from one user's assignee
	50	// rows, a handful, and sorting those is the cheap half.
	51	ordered bool
	52	}{
	53	{"DashboardIssues", queryPlan(t, s, dashboardIssuesQuery, int64(1)), "issues_recent", true},
	54	{"DashboardMRs", queryPlan(t, s, dashboardMRsQuery, int64(1)), "merge_requests_recent", true},
	55	{"ReviewQueue", queryPlan(t, s, reviewQueueQuery, int64(1)), "merge_requests_recent", true},
	56	{"AssignedIssues", queryPlan(t, s, assignedIssuesQuery, int64(1)), "issue_assignees_user", false},
	57	}
	58	for _, tc := range cases {
	59	if !strings.Contains(tc.plan, tc.want) {
	60	t.Errorf("%s does not use %s:\n%s", tc.name, tc.want, tc.plan)
	61	}
	62	if tc.ordered && strings.Contains(tc.plan, "USE TEMP B-TREE FOR ORDER BY") {
	63	t.Errorf("%s sorts instead of walking an index:\n%s", tc.name, tc.plan)
	64	}
	65	}
	66	}

internal/store/migrations/0035_dashboard_indexes.down.sql added +3

		@@ -0,0 +1,3 @@
	1	DROP INDEX issue_assignees_user;
	2	DROP INDEX merge_requests_recent;
	3	DROP INDEX issues_recent;

internal/store/migrations/0035_dashboard_indexes.up.sql added +15

		@@ -0,0 +1,15 @@
	1	-- The dashboard's lists are "the newest N rows I can reach". Reachability
	2	-- is a set of correlated subqueries, so no index can satisfy the filter;
	3	-- what an index can do is supply the order, so the walk stops at LIMIT
	4	-- instead of sorting every row in the table.
	5	--
	6	-- Deliberately not (state, updated_at): the merge request lists match
	7	-- state with IN, which turns one ordered walk into two that must be
	8	-- merged, and measured slower than no index at all — 12.1ms to 19.6ms on
	9	-- 20k rows. Ordering alone is what these queries want.
	10	CREATE INDEX issues_recent ON issues(updated_at DESC);
	11	CREATE INDEX merge_requests_recent ON merge_requests(updated_at DESC);
	12
	13	-- issue_assignees' primary key leads with issue_id, so AssignedIssues had
	14	-- no way in by user and tested EXISTS against every issue instead.
	15	CREATE INDEX issue_assignees_user ON issue_assignees(user_id);

Commit f686b22ad7

Commit `f686b22ad7`