Commit `aa224ebf3b`

aa224ebf3b824b965c3dd0da19c4bc202bb78396

parent: bc93d6cbca

Verified · cmc ci/build: success ci/test: success

cmc <hello@cleberg.net> · 2026-09-08 06:01 UTC

store, control, wiki: admin runners reports the build queue

`admin runners` now heads its output with the queue: builds pending
now, and over the last 24 hours the builds claimed, the average and
worst wait from creation to claim, and the builds the reaper ended
instead of a runner reporting them. Migration 0049 adds
`builds.reaped_at`, set by ReapStaleBuilds, so the last number can be
counted. The JSON is `{queue, runners}` where it was a bare list.

Ref #184

.gitbay/wiki/Admin.org +4 −1

		@@ -403,7 +403,10 @@ necessity, so the boundary is you choosing how to start it.
403	403
404	404	=gitbay dashboard= and =ssh git@<host> admin runners= list every account
405	405	that has polled as a runner: when it last polled, the =-repos= scope it
406		asked for, and the build it holds. A build a runner claimed and never
	406	asked for, and the build it holds. =admin runners= also heads the list
	407	with the queue: builds pending now, and over the last day how many were
	408	claimed, how long they waited to be claimed (average and worst), and
	409	how many the reaper ended instead of a runner reporting them. A build a runner claimed and never
407	410	reported is failed by the scheduler's minute tick, whether or not any
408	411	runner is still alive: within about two minutes of its log stream ending
409	412	with no outcome reported — the runner reports right after closing the

cmd/gitbay/main.go +1 −1

		@@ -111,7 +111,7 @@ func newRoot() *cobra.Command {
111	111	),
112	112	pass("invite", "issue a registration invite and mail its code: --email <address>", passOpts{server: []string{"admin", "invite"}}),
113	113	pass("stats", "instance statistics: counts and per-repository disk usage", passOpts{server: []string{"admin", "stats"}}),
114		pass("runners", "runner accounts: last poll, scope, the build each holds", passOpts{server: []string{"admin", "runners"}}),
	114	pass("runners", "the build queue and runner accounts: last poll, scope, the build each holds", passOpts{server: []string{"admin", "runners"}}),
115	115	group("repo", "any repository, for moderation (audited)",
116	116	pass("list", "every repository with size and last push: [--owner o] [--visibility v] [--limit n] [--cursor c]", passOpts{server: []string{"admin", "repo", "list"}}),
117	117	pass("archive", "archive a repository: <owner/name>", passOpts{server: []string{"admin", "repo", "archive"}}),

e2e/reap_test.go +11 −3

		@@ -80,6 +80,11 @@ func TestStaleBuildReapedWithoutRunner(t *testing.T) {
80	80	if out, _, _ := inst.ssh(t, aliceKey, "", "build", "log", "alice/app", "1"); !strings.Contains(out, "build abandoned") {
81	81	t.Fatalf("log lacks the abandonment note:\n%s", out)
82	82	}
	83	// The reaper's work is counted, apart from builds runners reported.
	84	if out, _, _ := inst.ssh(t, runnerKey, "", "admin", "runners", "--json"); !strings.Contains(out, `"reaped_24h":1`) \|\|
	85	!strings.Contains(out, `"claimed_24h":1`) {
	86	t.Fatalf("queue after reap:\n%s", out)
	87	}
83	88	}
84	89
85	90	func TestAdminRunners(t *testing.T) {
		@@ -93,7 +98,8 @@ func TestAdminRunners(t *testing.T) {
93	98	if _, _, code := inst.ssh(t, aliceKey, "", "admin", "runners"); code != 4 {
94	99	t.Fatal("non-admin listed runners")
95	100	}
96		if out, _, code := inst.ssh(t, rootKey, "", "admin", "runners", "--json"); code != 0 \|\| strings.TrimSpace(out) != `{"protocol_version":1,"data":[]}` {
	101	if out, _, code := inst.ssh(t, rootKey, "", "admin", "runners", "--json"); code != 0 \|\|
	102	strings.TrimSpace(out) != `{"protocol_version":1,"data":{"queue":{"pending":0,"claimed_24h":0,"claim_wait_avg_s":0,"claim_wait_max_s":0,"reaped_24h":0},"runners":[]}}` {
97	103	t.Fatalf("no runners yet: exit %d %s", code, out)
98	104	}
99	105	if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
		@@ -104,7 +110,8 @@ func TestAdminRunners(t *testing.T) {
104	110	t.Fatal("runner next failed")
105	111	}
106	112	out, _, _ := inst.ssh(t, rootKey, "", "admin", "runners")
107		if !strings.HasPrefix(out, "ci\t") \|\| !strings.Contains(out, "\talice/app\tidle") {
	113	if !strings.HasPrefix(out, "queue: 0 pending; last 24h: 0 claimed") \|\| !strings.Contains(out, "\nci\t") \|\|
	114	!strings.Contains(out, "\talice/app\tidle") {
108	115	t.Fatalf("idle runner row:\n%s", out)
109	116	}
110	117	work := t.TempDir()
		@@ -133,7 +140,8 @@ func TestAdminRunners(t *testing.T) {
133	140	if _, _, code := inst.ssh(t, runnerKey, "", "runner", "done", fmt.Sprint(claim.Data.ID), "success"); code != 0 {
134	141	t.Fatal("runner done failed")
135	142	}
136		if out, _, _ := inst.ssh(t, rootKey, "", "admin", "runners"); !strings.Contains(out, "\tany\tidle") {
	143	if out, _, _ := inst.ssh(t, rootKey, "", "admin", "runners"); !strings.Contains(out, "\tany\tidle") \|\|
	144	!strings.HasPrefix(out, "queue: 0 pending; last 24h: 1 claimed, wait avg ") {
137	145	t.Fatalf("runner still holds a build after done:\n%s", out)
138	146	}
139	147	// Host-local, the same read.

internal/control/admin.go +12 −2

		@@ -30,7 +30,7 @@ func init() {
30	30	Usage: "admin user demote <username>",
31	31	SSHOnly: true, Run: runAdminUserDemote})
32	32	register(Command{Path: []string{"admin", "runners"},
33		Summary: "runner accounts: last poll, scope, the build each holds (instance admins)",
	33	Summary: "the build queue and runner accounts: last poll, scope, the build each holds (instance admins)",
34	34	Usage: "admin runners",
35	35	ReadOnly: true, SSHOnly: true, Run: runAdminRunners})
36	36	register(Command{Path: []string{"admin", "repo", "list"},
		@@ -452,7 +452,17 @@ func runAdminRunners(c *Ctx, args []string) int {
452	452	if err != nil {
453	453	return c.fail(protocol.ExitFailure, "%v", err)
454	454	}
455		return c.emit(runners, func(w io.Writer) {
	455	queue, err := c.Store.QueueStats()
	456	if err != nil {
	457	return c.fail(protocol.ExitFailure, "%v", err)
	458	}
	459	if runners == nil {
	460	runners = []store.Runner{}
	461	}
	462	d := map[string]any{"queue": queue, "runners": runners}
	463	return c.emit(d, func(w io.Writer) {
	464	fmt.Fprintf(w, "queue: %d pending; last 24h: %d claimed, wait avg %ds max %ds, %d reaped\n",
	465	queue.Pending, queue.Claimed24h, queue.ClaimWaitAvgS, queue.ClaimWaitMaxS, queue.Reaped24h)
456	466	for _, r := range runners {
457	467	scope := r.Scope
458	468	if scope == "" {

internal/store/builds.go +28

		@@ -189,10 +189,38 @@ func (s *Store) ReapStaleBuilds() ([]Build, error) {
189	189	if err := s.FinishBuild(b.ID, "failure"); err != nil {
190	190	return nil, err
191	191	}
	192	if _, err := s.DB.Exec(`UPDATE builds SET reaped_at = finished_at WHERE id = ?`, b.ID); err != nil {
	193	return nil, err
	194	}
192	195	}
193	196	return stale, nil
194	197	}
195	198
	199	// QueueStats is the state of the build queue: what waits now, and over
	200	// the last day how long a build waited to be claimed and how many were
	201	// ended by the reaper rather than by a runner's report (#184).
	202	type QueueStats struct {
	203	Pending int64 `json:"pending"`
	204	Claimed24h int64 `json:"claimed_24h"`
	205	ClaimWaitAvgS int64 `json:"claim_wait_avg_s"`
	206	ClaimWaitMaxS int64 `json:"claim_wait_max_s"`
	207	Reaped24h int64 `json:"reaped_24h"`
	208	}
	209
	210	func (s *Store) QueueStats() (QueueStats, error) {
	211	var q QueueStats
	212	since := time.Now().UTC().Add(-24 * time.Hour).Format("2006-01-02T15:04:05Z")
	213	err := s.DB.QueryRow(`SELECT
	214	(SELECT COUNT(*) FROM builds WHERE status = 'pending'),
	215	COUNT(*),
	216	COALESCE(AVG(strftime('%s', started_at) - strftime('%s', created_at)), 0),
	217	COALESCE(MAX(strftime('%s', started_at) - strftime('%s', created_at)), 0),
	218	(SELECT COUNT(*) FROM builds WHERE reaped_at >= ?)
	219	FROM builds WHERE started_at >= ?`, since, since).
	220	Scan(&q.Pending, &q.Claimed24h, &q.ClaimWaitAvgS, &q.ClaimWaitMaxS, &q.Reaped24h)
	221	return q, err
	222	}
	223
196	224	// AppendBuildLog adds a chunk to the build's log, dropping bytes past the cap.
197	225	func (s *Store) AppendBuildLog(id int64, chunk []byte) error {
198	226	res, err := s.DB.Exec(`

internal/store/migrations/0049_build_reaped.down.sql added +1

		@@ -0,0 +1 @@
	1	ALTER TABLE builds DROP COLUMN reaped_at;

internal/store/migrations/0049_build_reaped.up.sql added +3

		@@ -0,0 +1,3 @@
	1	-- When the scheduler failed a build instead of its runner reporting it,
	2	-- so builds ended by the reaper can be counted (#184).
	3	ALTER TABLE builds ADD COLUMN reaped_at TEXT NOT NULL DEFAULT '';

Commit aa224ebf3b

Commit `aa224ebf3b`