Commit `16c2392828`

16c23928286fd79241df1f56ed29532fb793a439

parent: f08b7b5e1a

Verified · cmc

cmc <hello@cleberg.net> · 2026-09-20 11:10 UTC

store: report the push queue on the dashboard

Push was the one worker queue with no operator surface. It is also
the one whose commonest misconfiguration is invisible: validation can
check that the .p8 parses, not that key_id and team_id are the ones
Apple issued, so a transposed key_id starts clean and then
dead-letters every send on its first attempt with a log line and
nothing else.

Modelled on the mail block, with the device id where the recipient
is; a token is never echoed. Queues gains a field, so the JSON is
additive.

Ref #89

.gitbay/wiki/Admin.org +7 −1

		@@ -299,7 +299,7 @@ Every background worker keeps a backlog and a failure state. An instance
299	299	admin reads them all in one place:
300	300
301	301	#+begin_src sh
302		gitbay dashboard --json \| jq .queues # webhooks, mail, mirrors, builds, deps
	302	gitbay dashboard --json \| jq .queues # webhooks, mail, push, mirrors, builds, deps
303	303	#+end_src
304	304
305	305	Per worker: pending, retrying (pending with a failed attempt) and
		@@ -311,6 +311,12 @@ mirrors list the ones whose last sync failed;
311	311	dependency checks list the ones whose last check errored. Non-admins get
312	312	no =queues= key at all.
313	313
	314	Push rows name the device id, never the token. Watch this one after
	315	configuring =[push]=: a =key_id= or =team_id= Apple did not issue passes
	316	config validation, which can only check that the =.p8= parses, and then
	317	every send comes back =403 InvalidProviderToken= and dead-letters on its
	318	first attempt.
	319
314	320	In accounts mode the same read renders at =/admin=, linked from the rail
315	321	for admins. Anyone else gets a 404 there.
316	322

internal/control/dashboard.go +5

		@@ -203,6 +203,11 @@ func runDashboard(c *Ctx, args []string) int {
203	203	for _, it := range q.Mail.Items {
204	204	fmt.Fprintf(w, " %s\t%s\tattempts %d\t%s\n", it.Recipient, it.Subject, it.Attempts, it.LastError)
205	205	}
	206	// The device id, not the token: a token is never echoed.
	207	fmt.Fprintf(w, " push\tpending %d\tretrying %d\tfailed %d\n", q.Push.Pending, q.Push.Retrying, q.Push.Failed)
	208	for _, it := range q.Push.Items {
	209	fmt.Fprintf(w, " device %d\t%s\tattempts %d\t%s\n", it.DeviceID, it.Title, it.Attempts, it.LastError)
	210	}
206	211	fmt.Fprintf(w, " mirrors\tdirty %d\terrors %d\n", q.Mirrors.Dirty, q.Mirrors.Errors)
207	212	for _, it := range q.Mirrors.Items {
208	213	fmt.Fprintf(w, " %s\t%s\t%s\t%s\n", it.Repo, it.Direction, it.URL, it.LastError)

internal/control/dashboard_test.go added +53

		@@ -0,0 +1,53 @@
	1	package control
	2
	3	import (
	4	"bytes"
	5	"strings"
	6	"testing"
	7	)
	8
	9	// The push queue is the one worker queue whose worst failure — a key_id
	10	// or team_id Apple did not issue, which config validation cannot check —
	11	// dead-letters every row on its first attempt with nothing but a log
	12	// line. dashboard is where an admin would see that, so it reports push
	13	// beside the other five queues.
	14	func TestDashboardReportsThePushQueue(t *testing.T) {
	15	c := notifTestCtx(t, "cmc")
	16	c.User.IsAdmin = true
	17	uid := c.User.ID
	18	if _, err := c.Store.AddPushDevice(uid, "tok-a", "iphone"); err != nil {
	19	t.Fatal(err)
	20	}
	21	if err := c.Store.EnqueuePush(uid, "krz/gitbay", "cmc opened issue #1", "krz/gitbay/issues/1"); err != nil {
	22	t.Fatal(err)
	23	}
	24	due, err := c.Store.DuePush(20)
	25	if err != nil \|\| len(due) != 1 {
	26	t.Fatalf("DuePush: %v %+v", err, due)
	27	}
	28	if err := c.Store.MarkPushFailed(due[0].ID, "apns 403 InvalidProviderToken", nil); err != nil {
	29	t.Fatal(err)
	30	}
	31
	32	var out bytes.Buffer
	33	c.Stdout, c.Stderr = &out, &out
	34	if code := runDashboard(c, nil); code != 0 {
	35	t.Fatalf("exit %d: %s", code, out.String())
	36	}
	37	got := out.String()
	38	if !strings.Contains(got, "push\tpending 0\tretrying 0\tfailed 1") {
	39	t.Fatalf("no push queue row:\n%s", got)
	40	}
	41	if !strings.Contains(got, "apns 403 InvalidProviderToken") {
	42	t.Fatalf("dead-lettered row not listed:\n%s", got)
	43	}
	44
	45	out.Reset()
	46	c.JSON = true
	47	if code := runDashboard(c, nil); code != 0 {
	48	t.Fatalf("exit %d: %s", code, out.String())
	49	}
	50	if !strings.Contains(out.String(), `"push":{`) {
	51	t.Fatalf("no push key in the queues object:\n%s", out.String())
	52	}
	53	}

internal/store/queues.go +43

		@@ -11,6 +11,7 @@ type Queues struct {
11	11	Mirrors QueueMirrors `json:"mirrors"`
12	12	Builds QueueBuilds `json:"builds"`
13	13	Deps QueueDeps `json:"deps"`
	14	Push QueuePush `json:"push"`
14	15	}
15	16
16	17	type QueueWebhooks struct {
		@@ -50,6 +51,26 @@ type QueueMailRow struct {
50	51	CreatedAt string `json:"created_at"`
51	52	}
52	53
	54	// QueuePush is the APNs delivery queue, the mail queue's shape with the
	55	// device id where the recipient is: a device token is never echoed.
	56	type QueuePush struct {
	57	Pending int64 `json:"pending"`
	58	Retrying int64 `json:"retrying"`
	59	Failed int64 `json:"failed"`
	60	OldestPending string `json:"oldest_pending,omitempty"`
	61	Items []QueuePushRow `json:"items"`
	62	}
	63
	64	type QueuePushRow struct {
	65	ID int64 `json:"id"`
	66	DeviceID int64 `json:"device_id"`
	67	Title string `json:"title"`
	68	Attempts int64 `json:"attempts"`
	69	LastError string `json:"last_error,omitempty"`
	70	FailedAt string `json:"failed_at,omitempty"`
	71	CreatedAt string `json:"created_at"`
	72	}
	73
53	74	type QueueMirrors struct {
54	75	Dirty int64 `json:"dirty"` // waiting for a sync
55	76	Errors int64 `json:"errors"`
		@@ -112,6 +133,7 @@ func (s *Store) QueueStatus() (Queues, error) {
112	133	Mirrors: QueueMirrors{Items: []QueueMirrorRow{}},
113	134	Builds: QueueBuilds{Items: []QueueBuildRow{}},
114	135	Deps: QueueDeps{Items: []QueueDepRow{}},
	136	Push: QueuePush{Items: []QueuePushRow{}},
115	137	}
116	138
117	139	if err := s.DB.QueryRow(`SELECT
		@@ -191,6 +213,27 @@ func (s *Store) QueueStatus() (Queues, error) {
191	213	return q, err
192	214	}
193	215
	216	if err := s.DB.QueryRow(`SELECT
	217	COUNT(*) FILTER (WHERE sent_at IS NULL AND failed_at IS NULL),
	218	COUNT(*) FILTER (WHERE sent_at IS NULL AND failed_at IS NULL AND attempts > 0),
	219	COUNT(*) FILTER (WHERE failed_at IS NOT NULL),
	220	COALESCE(MIN(created_at) FILTER (WHERE sent_at IS NULL AND failed_at IS NULL), '')
	221	FROM push_queue`).Scan(&q.Push.Pending, &q.Push.Retrying, &q.Push.Failed, &q.Push.OldestPending); err != nil {
	222	return q, err
	223	}
	224	if err := s.queryEach(`SELECT id, device_id, title, attempts, COALESCE(last_error, ''), COALESCE(failed_at, ''), created_at
	225	FROM push_queue WHERE sent_at IS NULL AND (failed_at IS NOT NULL OR attempts > 0)
	226	ORDER BY id DESC LIMIT ?`, func(sc scanner) error {
	227	var p QueuePushRow
	228	if err := sc.Scan(&p.ID, &p.DeviceID, &p.Title, &p.Attempts, &p.LastError, &p.FailedAt, &p.CreatedAt); err != nil {
	229	return err
	230	}
	231	q.Push.Items = append(q.Push.Items, p)
	232	return nil
	233	}); err != nil {
	234	return q, err
	235	}
	236
194	237	if err := s.DB.QueryRow(`SELECT COUNT(*) FROM dep_checks WHERE last_error != ''`).Scan(&q.Deps.Errors); err != nil {
195	238	return q, err
196	239	}

internal/store/queues_test.go +71 −1

		@@ -1,6 +1,76 @@
1	1	package store
2	2
3		import "testing"
	3	import (
	4	"testing"
	5	"time"
	6	)
	7
	8	// Push is a worker queue like the others, and the one failure config
	9	// validation cannot catch — a key_id Apple did not issue — dead-letters
	10	// every row on its first attempt. Without a count and the rows here an
	11	// admin has no way to see that happening.
	12	func TestQueuesReportsPush(t *testing.T) {
	13	s := open(t)
	14	if err := s.MigrateUp(); err != nil {
	15	t.Fatal(err)
	16	}
	17	uid, err := s.CreateUser("cmc", true)
	18	if err != nil {
	19	t.Fatal(err)
	20	}
	21	id, err := s.AddPushDevice(uid, "tok-a", "iphone")
	22	if err != nil {
	23	t.Fatal(err)
	24	}
	25	for i := 0; i < 3; i++ {
	26	if err := s.EnqueuePush(uid, "krz/gitbay", "cmc opened issue #1", "krz/gitbay/issues/1"); err != nil {
	27	t.Fatal(err)
	28	}
	29	}
	30	due, err := s.DuePush(20)
	31	if err != nil {
	32	t.Fatal(err)
	33	}
	34	if len(due) != 3 {
	35	t.Fatalf("queued %d, want 3", len(due))
	36	}
	37	next := time.Now().Add(time.Minute)
	38	if err := s.MarkPushFailed(due[0].ID, "apns 503", &next); err != nil {
	39	t.Fatal(err)
	40	}
	41	if err := s.MarkPushFailed(due[1].ID, "apns 403 InvalidProviderToken", nil); err != nil {
	42	t.Fatal(err)
	43	}
	44
	45	q, err := s.QueueStatus()
	46	if err != nil {
	47	t.Fatal(err)
	48	}
	49	if q.Push.Pending != 2 \|\| q.Push.Retrying != 1 \|\| q.Push.Failed != 1 {
	50	t.Fatalf("counts: %+v", q.Push)
	51	}
	52	if q.Push.OldestPending == "" {
	53	t.Fatalf("no oldest pending: %+v", q.Push)
	54	}
	55	// Retrying and dead-lettered rows, newest first, as the mail queue
	56	// lists them.
	57	if len(q.Push.Items) != 2 {
	58	t.Fatalf("items: %+v", q.Push.Items)
	59	}
	60	if q.Push.Items[0].DeviceID != id \|\| q.Push.Items[0].FailedAt == "" \|\|
	61	q.Push.Items[0].LastError != "apns 403 InvalidProviderToken" {
	62	t.Fatalf("dead-lettered row: %+v", q.Push.Items[0])
	63	}
	64	if q.Push.Items[1].Attempts != 1 \|\| q.Push.Items[1].FailedAt != "" {
	65	t.Fatalf("retrying row: %+v", q.Push.Items[1])
	66	}
	67	// A device token is never echoed, here included.
	68	for _, it := range q.Push.Items {
	69	if it.Title != "krz/gitbay" \|\| it.CreatedAt == "" {
	70	t.Fatalf("row: %+v", it)
	71	}
	72	}
	73	}
4	74
5	75	// The build queue lists pending builds as well as running ones, so an
6	76	// admin can see what no runner is claiming without walking every repo.

Commit 16c2392828

Commit `16c2392828`