Commit 16c2392828
Verified · cmc
Layout: unified · split
.gitbay/wiki/Admin.org +7 −1
| @@ -299,7 +299,7 @@ Every background worker keeps a backlog and a failure state. An instance | |||
| 299 | admin reads them all in one place: | 299 | admin reads them all in one place: |
| 300 | 300 | ||
| 301 | #+begin_src sh | 301 | #+begin_src sh |
| 302 | gitbay dashboard --json | jq .queues # webhooks, mail, mirrors, builds, deps | 302 | gitbay dashboard --json | jq .queues # webhooks, mail, push, mirrors, builds, deps |
| 303 | #+end_src | 303 | #+end_src |
| 304 | 304 | ||
| 305 | Per worker: pending, retrying (pending with a failed attempt) and | 305 | Per worker: pending, retrying (pending with a failed attempt) and |
| @@ -311,6 +311,12 @@ mirrors list the ones whose last sync failed; | |||
| 311 | dependency checks list the ones whose last check errored. Non-admins get | 311 | dependency checks list the ones whose last check errored. Non-admins get |
| 312 | no =queues= key at all. | 312 | no =queues= key at all. |
| 313 | 313 | ||
| 314 | Push rows name the device id, never the token. Watch this one after | ||
| 315 | configuring =[push]=: a =key_id= or =team_id= Apple did not issue passes | ||
| 316 | config validation, which can only check that the =.p8= parses, and then | ||
| 317 | every send comes back =403 InvalidProviderToken= and dead-letters on its | ||
| 318 | first attempt. | ||
| 319 | |||
| 314 | In accounts mode the same read renders at =/admin=, linked from the rail | 320 | In accounts mode the same read renders at =/admin=, linked from the rail |
| 315 | for admins. Anyone else gets a 404 there. | 321 | for admins. Anyone else gets a 404 there. |
| 316 | 322 | ||
internal/control/dashboard.go +5
| @@ -203,6 +203,11 @@ func runDashboard(c *Ctx, args []string) int { | |||
| 203 | for _, it := range q.Mail.Items { | 203 | for _, it := range q.Mail.Items { |
| 204 | fmt.Fprintf(w, " %s\t%s\tattempts %d\t%s\n", it.Recipient, it.Subject, it.Attempts, it.LastError) | 204 | fmt.Fprintf(w, " %s\t%s\tattempts %d\t%s\n", it.Recipient, it.Subject, it.Attempts, it.LastError) |
| 205 | } | 205 | } |
| 206 | // The device id, not the token: a token is never echoed. | ||
| 207 | fmt.Fprintf(w, " push\tpending %d\tretrying %d\tfailed %d\n", q.Push.Pending, q.Push.Retrying, q.Push.Failed) | ||
| 208 | for _, it := range q.Push.Items { | ||
| 209 | fmt.Fprintf(w, " device %d\t%s\tattempts %d\t%s\n", it.DeviceID, it.Title, it.Attempts, it.LastError) | ||
| 210 | } | ||
| 206 | fmt.Fprintf(w, " mirrors\tdirty %d\terrors %d\n", q.Mirrors.Dirty, q.Mirrors.Errors) | 211 | fmt.Fprintf(w, " mirrors\tdirty %d\terrors %d\n", q.Mirrors.Dirty, q.Mirrors.Errors) |
| 207 | for _, it := range q.Mirrors.Items { | 212 | for _, it := range q.Mirrors.Items { |
| 208 | fmt.Fprintf(w, " %s\t%s\t%s\t%s\n", it.Repo, it.Direction, it.URL, it.LastError) | 213 | fmt.Fprintf(w, " %s\t%s\t%s\t%s\n", it.Repo, it.Direction, it.URL, it.LastError) |
internal/control/dashboard_test.go added +53
| @@ -0,0 +1,53 @@ | |||
| 1 | package control | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "bytes" | ||
| 5 | "strings" | ||
| 6 | "testing" | ||
| 7 | ) | ||
| 8 | |||
| 9 | // The push queue is the one worker queue whose worst failure — a key_id | ||
| 10 | // or team_id Apple did not issue, which config validation cannot check — | ||
| 11 | // dead-letters every row on its first attempt with nothing but a log | ||
| 12 | // line. dashboard is where an admin would see that, so it reports push | ||
| 13 | // beside the other five queues. | ||
| 14 | func TestDashboardReportsThePushQueue(t *testing.T) { | ||
| 15 | c := notifTestCtx(t, "cmc") | ||
| 16 | c.User.IsAdmin = true | ||
| 17 | uid := c.User.ID | ||
| 18 | if _, err := c.Store.AddPushDevice(uid, "tok-a", "iphone"); err != nil { | ||
| 19 | t.Fatal(err) | ||
| 20 | } | ||
| 21 | if err := c.Store.EnqueuePush(uid, "krz/gitbay", "cmc opened issue #1", "krz/gitbay/issues/1"); err != nil { | ||
| 22 | t.Fatal(err) | ||
| 23 | } | ||
| 24 | due, err := c.Store.DuePush(20) | ||
| 25 | if err != nil || len(due) != 1 { | ||
| 26 | t.Fatalf("DuePush: %v %+v", err, due) | ||
| 27 | } | ||
| 28 | if err := c.Store.MarkPushFailed(due[0].ID, "apns 403 InvalidProviderToken", nil); err != nil { | ||
| 29 | t.Fatal(err) | ||
| 30 | } | ||
| 31 | |||
| 32 | var out bytes.Buffer | ||
| 33 | c.Stdout, c.Stderr = &out, &out | ||
| 34 | if code := runDashboard(c, nil); code != 0 { | ||
| 35 | t.Fatalf("exit %d: %s", code, out.String()) | ||
| 36 | } | ||
| 37 | got := out.String() | ||
| 38 | if !strings.Contains(got, "push\tpending 0\tretrying 0\tfailed 1") { | ||
| 39 | t.Fatalf("no push queue row:\n%s", got) | ||
| 40 | } | ||
| 41 | if !strings.Contains(got, "apns 403 InvalidProviderToken") { | ||
| 42 | t.Fatalf("dead-lettered row not listed:\n%s", got) | ||
| 43 | } | ||
| 44 | |||
| 45 | out.Reset() | ||
| 46 | c.JSON = true | ||
| 47 | if code := runDashboard(c, nil); code != 0 { | ||
| 48 | t.Fatalf("exit %d: %s", code, out.String()) | ||
| 49 | } | ||
| 50 | if !strings.Contains(out.String(), `"push":{`) { | ||
| 51 | t.Fatalf("no push key in the queues object:\n%s", out.String()) | ||
| 52 | } | ||
| 53 | } | ||
internal/store/queues.go +43
| @@ -11,6 +11,7 @@ type Queues struct { | |||
| 11 | Mirrors QueueMirrors `json:"mirrors"` | 11 | Mirrors QueueMirrors `json:"mirrors"` |
| 12 | Builds QueueBuilds `json:"builds"` | 12 | Builds QueueBuilds `json:"builds"` |
| 13 | Deps QueueDeps `json:"deps"` | 13 | Deps QueueDeps `json:"deps"` |
| 14 | Push QueuePush `json:"push"` | ||
| 14 | } | 15 | } |
| 15 | 16 | ||
| 16 | type QueueWebhooks struct { | 17 | type QueueWebhooks struct { |
| @@ -50,6 +51,26 @@ type QueueMailRow struct { | |||
| 50 | CreatedAt string `json:"created_at"` | 51 | CreatedAt string `json:"created_at"` |
| 51 | } | 52 | } |
| 52 | 53 | ||
| 54 | // QueuePush is the APNs delivery queue, the mail queue's shape with the | ||
| 55 | // device id where the recipient is: a device token is never echoed. | ||
| 56 | type QueuePush struct { | ||
| 57 | Pending int64 `json:"pending"` | ||
| 58 | Retrying int64 `json:"retrying"` | ||
| 59 | Failed int64 `json:"failed"` | ||
| 60 | OldestPending string `json:"oldest_pending,omitempty"` | ||
| 61 | Items []QueuePushRow `json:"items"` | ||
| 62 | } | ||
| 63 | |||
| 64 | type QueuePushRow struct { | ||
| 65 | ID int64 `json:"id"` | ||
| 66 | DeviceID int64 `json:"device_id"` | ||
| 67 | Title string `json:"title"` | ||
| 68 | Attempts int64 `json:"attempts"` | ||
| 69 | LastError string `json:"last_error,omitempty"` | ||
| 70 | FailedAt string `json:"failed_at,omitempty"` | ||
| 71 | CreatedAt string `json:"created_at"` | ||
| 72 | } | ||
| 73 | |||
| 53 | type QueueMirrors struct { | 74 | type QueueMirrors struct { |
| 54 | Dirty int64 `json:"dirty"` // waiting for a sync | 75 | Dirty int64 `json:"dirty"` // waiting for a sync |
| 55 | Errors int64 `json:"errors"` | 76 | Errors int64 `json:"errors"` |
| @@ -112,6 +133,7 @@ func (s *Store) QueueStatus() (Queues, error) { | |||
| 112 | Mirrors: QueueMirrors{Items: []QueueMirrorRow{}}, | 133 | Mirrors: QueueMirrors{Items: []QueueMirrorRow{}}, |
| 113 | Builds: QueueBuilds{Items: []QueueBuildRow{}}, | 134 | Builds: QueueBuilds{Items: []QueueBuildRow{}}, |
| 114 | Deps: QueueDeps{Items: []QueueDepRow{}}, | 135 | Deps: QueueDeps{Items: []QueueDepRow{}}, |
| 136 | Push: QueuePush{Items: []QueuePushRow{}}, | ||
| 115 | } | 137 | } |
| 116 | 138 | ||
| 117 | if err := s.DB.QueryRow(`SELECT | 139 | if err := s.DB.QueryRow(`SELECT |
| @@ -191,6 +213,27 @@ func (s *Store) QueueStatus() (Queues, error) { | |||
| 191 | return q, err | 213 | return q, err |
| 192 | } | 214 | } |
| 193 | 215 | ||
| 216 | if err := s.DB.QueryRow(`SELECT | ||
| 217 | COUNT(*) FILTER (WHERE sent_at IS NULL AND failed_at IS NULL), | ||
| 218 | COUNT(*) FILTER (WHERE sent_at IS NULL AND failed_at IS NULL AND attempts > 0), | ||
| 219 | COUNT(*) FILTER (WHERE failed_at IS NOT NULL), | ||
| 220 | COALESCE(MIN(created_at) FILTER (WHERE sent_at IS NULL AND failed_at IS NULL), '') | ||
| 221 | FROM push_queue`).Scan(&q.Push.Pending, &q.Push.Retrying, &q.Push.Failed, &q.Push.OldestPending); err != nil { | ||
| 222 | return q, err | ||
| 223 | } | ||
| 224 | if err := s.queryEach(`SELECT id, device_id, title, attempts, COALESCE(last_error, ''), COALESCE(failed_at, ''), created_at | ||
| 225 | FROM push_queue WHERE sent_at IS NULL AND (failed_at IS NOT NULL OR attempts > 0) | ||
| 226 | ORDER BY id DESC LIMIT ?`, func(sc scanner) error { | ||
| 227 | var p QueuePushRow | ||
| 228 | if err := sc.Scan(&p.ID, &p.DeviceID, &p.Title, &p.Attempts, &p.LastError, &p.FailedAt, &p.CreatedAt); err != nil { | ||
| 229 | return err | ||
| 230 | } | ||
| 231 | q.Push.Items = append(q.Push.Items, p) | ||
| 232 | return nil | ||
| 233 | }); err != nil { | ||
| 234 | return q, err | ||
| 235 | } | ||
| 236 | |||
| 194 | if err := s.DB.QueryRow(`SELECT COUNT(*) FROM dep_checks WHERE last_error != ''`).Scan(&q.Deps.Errors); err != nil { | 237 | if err := s.DB.QueryRow(`SELECT COUNT(*) FROM dep_checks WHERE last_error != ''`).Scan(&q.Deps.Errors); err != nil { |
| 195 | return q, err | 238 | return q, err |
| 196 | } | 239 | } |
internal/store/queues_test.go +71 −1
| @@ -1,6 +1,76 @@ | |||
| 1 | package store | 1 | package store |
| 2 | 2 | ||
| 3 | import "testing" | 3 | import ( |
| 4 | "testing" | ||
| 5 | "time" | ||
| 6 | ) | ||
| 7 | |||
| 8 | // Push is a worker queue like the others, and the one failure config | ||
| 9 | // validation cannot catch — a key_id Apple did not issue — dead-letters | ||
| 10 | // every row on its first attempt. Without a count and the rows here an | ||
| 11 | // admin has no way to see that happening. | ||
| 12 | func TestQueuesReportsPush(t *testing.T) { | ||
| 13 | s := open(t) | ||
| 14 | if err := s.MigrateUp(); err != nil { | ||
| 15 | t.Fatal(err) | ||
| 16 | } | ||
| 17 | uid, err := s.CreateUser("cmc", true) | ||
| 18 | if err != nil { | ||
| 19 | t.Fatal(err) | ||
| 20 | } | ||
| 21 | id, err := s.AddPushDevice(uid, "tok-a", "iphone") | ||
| 22 | if err != nil { | ||
| 23 | t.Fatal(err) | ||
| 24 | } | ||
| 25 | for i := 0; i < 3; i++ { | ||
| 26 | if err := s.EnqueuePush(uid, "krz/gitbay", "cmc opened issue #1", "krz/gitbay/issues/1"); err != nil { | ||
| 27 | t.Fatal(err) | ||
| 28 | } | ||
| 29 | } | ||
| 30 | due, err := s.DuePush(20) | ||
| 31 | if err != nil { | ||
| 32 | t.Fatal(err) | ||
| 33 | } | ||
| 34 | if len(due) != 3 { | ||
| 35 | t.Fatalf("queued %d, want 3", len(due)) | ||
| 36 | } | ||
| 37 | next := time.Now().Add(time.Minute) | ||
| 38 | if err := s.MarkPushFailed(due[0].ID, "apns 503", &next); err != nil { | ||
| 39 | t.Fatal(err) | ||
| 40 | } | ||
| 41 | if err := s.MarkPushFailed(due[1].ID, "apns 403 InvalidProviderToken", nil); err != nil { | ||
| 42 | t.Fatal(err) | ||
| 43 | } | ||
| 44 | |||
| 45 | q, err := s.QueueStatus() | ||
| 46 | if err != nil { | ||
| 47 | t.Fatal(err) | ||
| 48 | } | ||
| 49 | if q.Push.Pending != 2 || q.Push.Retrying != 1 || q.Push.Failed != 1 { | ||
| 50 | t.Fatalf("counts: %+v", q.Push) | ||
| 51 | } | ||
| 52 | if q.Push.OldestPending == "" { | ||
| 53 | t.Fatalf("no oldest pending: %+v", q.Push) | ||
| 54 | } | ||
| 55 | // Retrying and dead-lettered rows, newest first, as the mail queue | ||
| 56 | // lists them. | ||
| 57 | if len(q.Push.Items) != 2 { | ||
| 58 | t.Fatalf("items: %+v", q.Push.Items) | ||
| 59 | } | ||
| 60 | if q.Push.Items[0].DeviceID != id || q.Push.Items[0].FailedAt == "" || | ||
| 61 | q.Push.Items[0].LastError != "apns 403 InvalidProviderToken" { | ||
| 62 | t.Fatalf("dead-lettered row: %+v", q.Push.Items[0]) | ||
| 63 | } | ||
| 64 | if q.Push.Items[1].Attempts != 1 || q.Push.Items[1].FailedAt != "" { | ||
| 65 | t.Fatalf("retrying row: %+v", q.Push.Items[1]) | ||
| 66 | } | ||
| 67 | // A device token is never echoed, here included. | ||
| 68 | for _, it := range q.Push.Items { | ||
| 69 | if it.Title != "krz/gitbay" || it.CreatedAt == "" { | ||
| 70 | t.Fatalf("row: %+v", it) | ||
| 71 | } | ||
| 72 | } | ||
| 73 | } | ||
| 4 | 74 | ||
| 5 | // The build queue lists pending builds as well as running ones, so an | 75 | // The build queue lists pending builds as well as running ones, so an |
| 6 | // admin can see what no runner is claiming without walking every repo. | 76 | // admin can see what no runner is claiming without walking every repo. |