Commit `9cdfb3a89d`

9cdfb3a89de20278ac7db640fd3c85c46559b91e

parent: 8ff79f1044

Verified · cmc

cmc <hello@cleberg.net> · 2026-08-24 00:53 UTC

webhooks: signed outbound delivery with retries and dead-lettering

- migration 0005: webhooks (per-repo url/secret/event filter) and
  webhook_deliveries (attempts, schedule, outcome); RecordEvent
  enqueues deliveries for matching active hooks in the same
  transaction; post-receive now records push events (ref/old/new/
  forced/deleted), so pushes are webhook-visible
- deliverer goroutine polls due deliveries: HMAC-SHA256 signature
  (X-Gitbay-Signature-256) when a secret is set, X-Gitbay-Event/
  -Delivery headers, 10s timeout, no redirects; non-2xx or transport
  errors retry with exponential backoff (base 30s, doubling), dead-
  lettered after 5 attempts; GITBAY_WEBHOOK_RETRY_BASE test knob
- SSRF: targets validated at add time and re-checked at dial time
  (loopback/private/link-local refused unless [webhooks] allow_local);
  http/https only; redirects never followed
- webhook add/list/remove/deliveries/redeliver control commands (repo
  admin) and CLI passthrough group; redeliver revives dead letters
- e2e: HMAC verification, payload shape, push events, event filters,
  fail-twice-then-succeed with attempts recorded, dead-letter and
  manual redelivery, add-time SSRF and scheme rejection

Layout: unified · split

cmd/gitbay/main.go +11

		@@ -31,6 +31,7 @@ func main() {
31	31	mrCmd(),
32	32	webCmd(),
33	33	orgCmd(),
	34	webhookCmd(),
34	35	remoteCmd(),
35	36	initCmd(),
36	37	pass("register", "create an account on the default instance: gitbay register --username <n> --email <a> \| --invite <code>",
		@@ -294,6 +295,16 @@ func webCmd() *cobra.Command {
294	295	)
295	296	}
296	297
	298	func webhookCmd() *cobra.Command {
	299	return group("webhook", "outbound event delivery",
	300	pass("add", "add a webhook: <url> [--secret s] [--events k1,k2\|*]", passOpts{server: []string{"webhook", "add"}, needsRepo: true}),
	301	pass("list", "list webhooks", passOpts{server: []string{"webhook", "list"}, needsRepo: true}),
	302	pass("remove", "remove a webhook: <id>", passOpts{server: []string{"webhook", "remove"}, needsRepo: true}),
	303	pass("deliveries", "recent deliveries [--limit n]", passOpts{server: []string{"webhook", "deliveries"}, needsRepo: true}),
	304	pass("redeliver", "requeue a delivery: <delivery-id>", passOpts{server: []string{"webhook", "redeliver"}, needsRepo: true}),
	305	)
	306	}
	307
297	308	func orgCmd() *cobra.Command {
298	309	return group("org", "organizations",
299	310	pass("create", "create an organization", passOpts{server: []string{"org", "create"}}),

cmd/gitbayd/main.go +15

		@@ -3,6 +3,7 @@
3	3	package main
4	4
5	5	import (
	6	"context"
6	7	"fmt"
7	8	"log/slog"
8	9	"net"
		@@ -11,6 +12,7 @@ import (
11	12	"path/filepath"
12	13	"strconv"
13	14	"strings"
	15	"time"
14	16
15	17	"github.com/spf13/cobra"
16	18	"golang.org/x/crypto/acme/autocert"
		@@ -25,6 +27,7 @@ import (
25	27	"gitbay.org/gitbay/internal/policy"
26	28	"gitbay.org/gitbay/internal/sshd"
27	29	"gitbay.org/gitbay/internal/store"
	30	"gitbay.org/gitbay/internal/webhook"
28	31	)
29	32
30	33	func openStore(cfg config.Config) (*store.Store, error) {
		@@ -119,6 +122,18 @@ func serveCmd() *cobra.Command {
119	122	}
120	123	defer stopHookd()
121	124
	125	// Outbound webhook deliveries. The retry base is overridable
	126	// for tests via GITBAY_WEBHOOK_RETRY_BASE.
	127	retryBase := 30 * time.Second
	128	if v := os.Getenv("GITBAY_WEBHOOK_RETRY_BASE"); v != "" {
	129	if d, err := time.ParseDuration(v); err == nil {
	130	retryBase = d
	131	}
	132	}
	133	whCtx, whCancel := context.WithCancel(context.Background())
	134	defer whCancel()
	135	go webhook.New(st, cfg.Webhooks.AllowLocal, retryBase).Run(whCtx)
	136
122	137	errCh := make(chan error, 3)
123	138	if cfg.SSH.Mode == "embedded" {
124	139	srv, err := sshd.New(cfg, st)

e2e/webhook_test.go added +256

		@@ -0,0 +1,256 @@
	1	package e2e
	2
	3	import (
	4	"crypto/hmac"
	5	"crypto/sha256"
	6	"encoding/hex"
	7	"encoding/json"
	8	"fmt"
	9	"io"
	10	"net"
	11	"net/http"
	12	"os"
	13	"os/exec"
	14	"strings"
	15	"sync"
	16	"testing"
	17	"time"
	18	)
	19
	20	// hookReceiver captures webhook deliveries and can be told to fail.
	21	type hookReceiver struct {
	22	addr string
	23	mu sync.Mutex
	24	got []capturedHook
	25	failNext int // respond 500 to this many requests
	26	}
	27
	28	type capturedHook struct {
	29	event string
	30	delivery string
	31	signature string
	32	body []byte
	33	}
	34
	35	func startHookReceiver(t testing.T) hookReceiver {
	36	t.Helper()
	37	ln, err := net.Listen("tcp", "127.0.0.1:0")
	38	if err != nil {
	39	t.Fatal(err)
	40	}
	41	t.Cleanup(func() { ln.Close() })
	42	h := &hookReceiver{addr: ln.Addr().String()}
	43	go http.Serve(ln, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	44	body, _ := io.ReadAll(r.Body)
	45	h.mu.Lock()
	46	defer h.mu.Unlock()
	47	if h.failNext > 0 {
	48	h.failNext--
	49	w.WriteHeader(500)
	50	return
	51	}
	52	h.got = append(h.got, capturedHook{
	53	event: r.Header.Get("X-Gitbay-Event"),
	54	delivery: r.Header.Get("X-Gitbay-Delivery"),
	55	signature: r.Header.Get("X-Gitbay-Signature-256"),
	56	body: body,
	57	})
	58	w.WriteHeader(204)
	59	}))
	60	return h
	61	}
	62
	63	func (h hookReceiver) waitN(t testing.T, n int) []capturedHook {
	64	t.Helper()
	65	deadline := time.Now().Add(15 * time.Second)
	66	for time.Now().Before(deadline) {
	67	h.mu.Lock()
	68	if len(h.got) >= n {
	69	out := append([]capturedHook(nil), h.got...)
	70	h.mu.Unlock()
	71	return out
	72	}
	73	h.mu.Unlock()
	74	time.Sleep(100 * time.Millisecond)
	75	}
	76	t.Fatalf("only %d deliveries arrived, want %d", len(h.got), n)
	77	return nil
	78	}
	79
	80	func TestWebhooks(t *testing.T) {
	81	inst := startInstanceWith(t, "[webhooks]\nallow_local = true\n")
	82	// Restart the daemon with a fast retry base for the failure tests.
	83	inst.proc.Process.Kill()
	84	inst.proc.Wait()
	85	inst.proc = exec.Command(inst.gitbayd, "--config", inst.config, "serve")
	86	inst.proc.Env = append(os.Environ(), "GITBAY_WEBHOOK_RETRY_BASE=500ms")
	87	inst.proc.Stderr = os.Stderr
	88	if err := inst.proc.Start(); err != nil {
	89	t.Fatal(err)
	90	}
	91	t.Cleanup(func() { inst.proc.Process.Kill(); inst.proc.Wait() })
	92	deadline := time.Now().Add(10 * time.Second)
	93	for {
	94	conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", inst.port), 200*time.Millisecond)
	95	if err == nil {
	96	conn.Close()
	97	break
	98	}
	99	if time.Now().After(deadline) {
	100	t.Fatal("daemon did not restart")
	101	}
	102	time.Sleep(50 * time.Millisecond)
	103	}
	104
	105	aliceKey := inst.newKey(t, "alice")
	106	inst.admin(t, "admin", "user", "create", "alice",
	107	"--key", aliceKey+".pub", "--email", "alice@example.test", "--verified")
	108	if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/proj"); code != 0 {
	109	t.Fatalf("repo create: %s", errOut)
	110	}
	111
	112	recv := startHookReceiver(t)
	113	hookURL := "http://" + recv.addr + "/hook"
	114	if _, errOut, code := inst.ssh(t, aliceKey, "",
	115	"webhook", "add", "alice/proj", hookURL, "--secret", "s3cret"); code != 0 {
	116	t.Fatalf("webhook add: %s", errOut)
	117	}
	118
	119	// An issue event arrives, signed and shaped.
	120	if _, _, code := inst.ssh(t, aliceKey, "", "issue", "create", "alice/proj", "--title", "'hook me'"); code != 0 {
	121	t.Fatal("issue create failed")
	122	}
	123	got := recv.waitN(t, 1)
	124	h := got[0]
	125	if h.event != "issue.created" \|\| h.delivery == "" {
	126	t.Fatalf("delivery headers: %+v", h)
	127	}
	128	mac := hmac.New(sha256.New, []byte("s3cret"))
	129	mac.Write(h.body)
	130	if h.signature != "sha256="+hex.EncodeToString(mac.Sum(nil)) {
	131	t.Fatalf("HMAC mismatch: %s", h.signature)
	132	}
	133	var p struct {
	134	Event string `json:"event"`
	135	Repo string `json:"repo"`
	136	Actor string `json:"actor"`
	137	Data struct {
	138	Number int `json:"number"`
	139	} `json:"data"`
	140	}
	141	if err := json.Unmarshal(h.body, &p); err != nil {
	142	t.Fatalf("payload: %v\n%s", err, h.body)
	143	}
	144	if p.Repo != "alice/proj" \|\| p.Actor != "alice" \|\| p.Data.Number != 1 {
	145	t.Fatalf("payload fields: %+v", p)
	146	}
	147
	148	// Push events flow through the hook chain.
	149	work := t.TempDir()
	150	env := inst.gitEnv(aliceKey)
	151	mustGit(t, work, env, "clone", inst.sshURL("alice/proj"), "w")
	152	dir := work + "/w"
	153	os.WriteFile(dir+"/f.txt", []byte("x\n"), 0o644)
	154	mustGit(t, dir, env, "checkout", "-q", "-b", "main")
	155	mustGit(t, dir, env, "add", ".")
	156	mustGit(t, dir, env, "commit", "-q", "-m", "push event")
	157	mustGit(t, dir, env, "push", "-q", "origin", "main")
	158	got = recv.waitN(t, 2)
	159	push := got[1]
	160	if push.event != "push" \|\| !strings.Contains(string(push.body), `"ref":"refs/heads/main"`) {
	161	t.Fatalf("push event: %s %s", push.event, push.body)
	162	}
	163
	164	// Event filters: a hook subscribed to mr.created ignores issues.
	165	recv2 := startHookReceiver(t)
	166	if _, _, code := inst.ssh(t, aliceKey, "",
	167	"webhook", "add", "alice/proj", "http://"+recv2.addr+"/", "--events", "mr.created"); code != 0 {
	168	t.Fatal("filtered webhook add failed")
	169	}
	170	if _, _, code := inst.ssh(t, aliceKey, "", "issue", "create", "alice/proj", "--title", "'no hook'"); code != 0 {
	171	t.Fatal("issue 2 failed")
	172	}
	173	got = recv.waitN(t, 3) // unfiltered hook sees it
	174	if got[2].event != "issue.created" {
	175	t.Fatalf("third delivery: %s", got[2].event)
	176	}
	177	time.Sleep(500 * time.Millisecond)
	178	recv2.mu.Lock()
	179	if len(recv2.got) != 0 {
	180	t.Fatalf("filtered hook received %d deliveries", len(recv2.got))
	181	}
	182	recv2.mu.Unlock()
	183
	184	// Retries: fail twice, then succeed; attempts recorded.
	185	recv.mu.Lock()
	186	recv.failNext = 2
	187	recv.mu.Unlock()
	188	if _, _, code := inst.ssh(t, aliceKey, "", "issue", "close", "alice/proj", "1"); code != 0 {
	189	t.Fatal("close failed")
	190	}
	191	got = recv.waitN(t, 4)
	192	if got[3].event != "issue.closed" {
	193	t.Fatalf("retried event: %s", got[3].event)
	194	}
	195	out, _, _ := inst.ssh(t, aliceKey, "", "webhook", "deliveries", "alice/proj", "--json")
	196	if !strings.Contains(out, `"attempts":3`) {
	197	t.Fatalf("retry attempts not recorded:\n%s", out)
	198	}
	199
	200	// Dead-letter after max attempts, then manual redelivery revives it.
	201	recv.mu.Lock()
	202	recv.failNext = 99
	203	recv.mu.Unlock()
	204	if _, _, code := inst.ssh(t, aliceKey, "", "issue", "reopen", "alice/proj", "1"); code != 0 {
	205	t.Fatal("reopen failed")
	206	}
	207	var deadID string
	208	deadlineDL := time.Now().Add(30 * time.Second)
	209	for time.Now().Before(deadlineDL) {
	210	out, _, _ = inst.ssh(t, aliceKey, "", "webhook", "deliveries", "alice/proj", "--json")
	211	var envl struct {
	212	Data []struct {
	213	ID int64 `json:"id"`
	214	Event string `json:"event"`
	215	Status string `json:"status"`
	216	} `json:"data"`
	217	}
	218	json.Unmarshal([]byte(out), &envl)
	219	for _, d := range envl.Data {
	220	if d.Event == "issue.open" && d.Status == "failed" {
	221	deadID = fmt.Sprint(d.ID)
	222	}
	223	}
	224	if deadID != "" {
	225	break
	226	}
	227	time.Sleep(300 * time.Millisecond)
	228	}
	229	if deadID == "" {
	230	t.Fatalf("delivery never dead-lettered:\n%s", out)
	231	}
	232	recv.mu.Lock()
	233	recv.failNext = 0
	234	prev := len(recv.got)
	235	recv.mu.Unlock()
	236	if _, errOut, code := inst.ssh(t, aliceKey, "", "webhook", "redeliver", "alice/proj", deadID); code != 0 {
	237	t.Fatalf("redeliver: %s", errOut)
	238	}
	239	recv.waitN(t, prev+1)
	240
	241	// SSRF: on a default instance (allow_local off), local targets are
	242	// rejected at add time.
	243	inst2 := startInstance(t)
	244	k2 := inst2.newKey(t, "a2")
	245	inst2.admin(t, "admin", "user", "create", "a2", "--key", k2+".pub")
	246	if _, _, code := inst2.ssh(t, k2, "", "repo", "create", "a2/r"); code != 0 {
	247	t.Fatal("repo create failed")
	248	}
	249	_, errOut, code := inst2.ssh(t, k2, "", "webhook", "add", "a2/r", "http://127.0.0.1:9/x")
	250	if code != 2 \|\| !strings.Contains(errOut, "SSRF") {
	251	t.Fatalf("local webhook target accepted: exit %d, %s", code, errOut)
	252	}
	253	if _, _, code := inst2.ssh(t, k2, "", "webhook", "add", "a2/r", "ftp://example.com/x"); code != 2 {
	254	t.Fatal("non-http scheme accepted")
	255	}
	256	}

internal/config/config.go +7

		@@ -20,6 +20,7 @@ type Config struct {
20	20	Web Web `toml:"web"`
21	21	Registration Registration `toml:"registration"`
22	22	API API `toml:"api"`
	23	Webhooks Webhooks `toml:"webhooks"`
23	24	Limits Limits `toml:"limits"`
24	25	Mail Mail `toml:"mail"`
25	26	}
		@@ -69,6 +70,12 @@ type API struct {
69	70	Enabled bool `toml:"enabled"`
70	71	}
71	72
	73	// Webhooks controls outbound delivery. AllowLocal permits endpoints on
	74	// loopback/private addresses (off by default: SSRF).
	75	type Webhooks struct {
	76	AllowLocal bool `toml:"allow_local"`
	77	}
	78
72	79	type Limits struct {
73	80	MaxPackBytes int64 `toml:"max_pack_bytes"`
74	81	MaxBlobBytes int64 `toml:"max_blob_bytes"`

internal/control/webhook.go added +202

		@@ -0,0 +1,202 @@
	1	package control
	2
	3	import (
	4	"errors"
	5	"fmt"
	6	"io"
	7	"strconv"
	8
	9	"gitbay.org/gitbay/internal/policy"
	10	"gitbay.org/gitbay/internal/protocol"
	11	"gitbay.org/gitbay/internal/store"
	12	"gitbay.org/gitbay/internal/webhook"
	13	)
	14
	15	func init() {
	16	register(Command{Path: []string{"webhook", "add"},
	17	Summary: "add a webhook: webhook add <owner/name> <url> [--secret <s>] [--events push,issue.created\|*]", Run: runWebhookAdd})
	18	register(Command{Path: []string{"webhook", "list"},
	19	Summary: "list webhooks: webhook list <owner/name>", ReadOnly: true, Run: runWebhookList})
	20	register(Command{Path: []string{"webhook", "remove"},
	21	Summary: "remove a webhook: webhook remove <owner/name> <id>", Run: runWebhookRemove})
	22	register(Command{Path: []string{"webhook", "deliveries"},
	23	Summary: "recent deliveries: webhook deliveries <owner/name> [--limit n]", ReadOnly: true, Run: runWebhookDeliveries})
	24	register(Command{Path: []string{"webhook", "redeliver"},
	25	Summary: "queue a delivery again: webhook redeliver <owner/name> <delivery-id>", Run: runWebhookRedeliver})
	26	}
	27
	28	func runWebhookAdd(c *Ctx, args []string) int {
	29	var path, url, secret string
	30	events := "*"
	31	for i := 0; i < len(args); i++ {
	32	switch args[i] {
	33	case "--secret", "--events":
	34	if i+1 >= len(args) {
	35	return c.fail(protocol.ExitUsage, "%s requires a value", args[i])
	36	}
	37	if args[i] == "--secret" {
	38	secret = args[i+1]
	39	} else {
	40	events = args[i+1]
	41	}
	42	i++
	43	default:
	44	if path == "" {
	45	path = args[i]
	46	} else if url == "" {
	47	url = args[i]
	48	} else {
	49	return c.fail(protocol.ExitUsage, "unexpected argument %q", args[i])
	50	}
	51	}
	52	}
	53	if path == "" \|\| url == "" {
	54	return c.fail(protocol.ExitUsage, "usage: webhook add <owner/name> <url> [--secret <s>] [--events <k1,k2>\|*]")
	55	}
	56	repo, code := resolveRepo(c, path, policy.CanAdmin)
	57	if code >= 0 {
	58	return code
	59	}
	60	if err := webhook.ValidateURL(url, c.Cfg.Webhooks.AllowLocal); err != nil {
	61	return c.fail(protocol.ExitUsage, "%v", err)
	62	}
	63	id, err := c.Store.AddWebhook(repo.ID, url, secret, events)
	64	if err != nil {
	65	return c.fail(protocol.ExitFailure, "%v", err)
	66	}
	67	return c.emit(map[string]any{"id": id, "url": url, "events": events}, func(w io.Writer) {
	68	fmt.Fprintf(w, "webhook %d added for %s (%s)\n", id, repo.Path(), events)
	69	})
	70	}
	71
	72	func runWebhookList(c *Ctx, args []string) int {
	73	if len(args) != 1 {
	74	return c.fail(protocol.ExitUsage, "usage: webhook list <owner/name>")
	75	}
	76	repo, code := resolveRepo(c, args[0], policy.CanAdmin)
	77	if code >= 0 {
	78	return code
	79	}
	80	hooks, err := c.Store.ListWebhooks(repo.ID)
	81	if err != nil {
	82	return c.fail(protocol.ExitFailure, "%v", err)
	83	}
	84	type out struct {
	85	ID int64 `json:"id"`
	86	URL string `json:"url"`
	87	Events string `json:"events"`
	88	Active bool `json:"active"`
	89	Secret bool `json:"has_secret"`
	90	}
	91	var ds []out
	92	for _, h := range hooks {
	93	ds = append(ds, out{h.ID, h.URL, h.Events, h.Active, h.Secret != ""})
	94	}
	95	return c.emit(ds, func(w io.Writer) {
	96	for _, d := range ds {
	97	fmt.Fprintf(w, "%d\t%s\t%s\n", d.ID, d.URL, d.Events)
	98	}
	99	})
	100	}
	101
	102	func runWebhookRemove(c *Ctx, args []string) int {
	103	if len(args) != 2 {
	104	return c.fail(protocol.ExitUsage, "usage: webhook remove <owner/name> <id>")
	105	}
	106	repo, code := resolveRepo(c, args[0], policy.CanAdmin)
	107	if code >= 0 {
	108	return code
	109	}
	110	id, err := strconv.ParseInt(args[1], 10, 64)
	111	if err != nil {
	112	return c.fail(protocol.ExitUsage, "bad webhook id %q", args[1])
	113	}
	114	if err := c.Store.RemoveWebhook(repo.ID, id); err != nil {
	115	if errors.Is(err, store.ErrNotFound) {
	116	return c.fail(protocol.ExitNotFound, "no webhook %d on %s", id, repo.Path())
	117	}
	118	return c.fail(protocol.ExitFailure, "%v", err)
	119	}
	120	return c.emit(map[string]any{"removed": id}, func(w io.Writer) {
	121	fmt.Fprintf(w, "removed webhook %d\n", id)
	122	})
	123	}
	124
	125	func runWebhookDeliveries(c *Ctx, args []string) int {
	126	limit := 20
	127	var path string
	128	for i := 0; i < len(args); i++ {
	129	if args[i] == "--limit" {
	130	if i+1 >= len(args) {
	131	return c.fail(protocol.ExitUsage, "--limit requires a value")
	132	}
	133	n, err := strconv.Atoi(args[i+1])
	134	if err != nil \|\| n < 1 \|\| n > 200 {
	135	return c.fail(protocol.ExitUsage, "--limit must be 1..200")
	136	}
	137	limit = n
	138	i++
	139	continue
	140	}
	141	if path != "" {
	142	return c.fail(protocol.ExitUsage, "usage: webhook deliveries <owner/name> [--limit n]")
	143	}
	144	path = args[i]
	145	}
	146	if path == "" {
	147	return c.fail(protocol.ExitUsage, "usage: webhook deliveries <owner/name> [--limit n]")
	148	}
	149	repo, code := resolveRepo(c, path, policy.CanAdmin)
	150	if code >= 0 {
	151	return code
	152	}
	153	ds, err := c.Store.ListDeliveries(repo.ID, limit)
	154	if err != nil {
	155	return c.fail(protocol.ExitFailure, "%v", err)
	156	}
	157	type out struct {
	158	ID int64 `json:"id"`
	159	URL string `json:"url"`
	160	Event string `json:"event"`
	161	Status string `json:"status"`
	162	Attempts int `json:"attempts"`
	163	LastStatus int `json:"last_status,omitempty"`
	164	LastError string `json:"last_error,omitempty"`
	165	}
	166	var rows []out
	167	for _, d := range ds {
	168	rows = append(rows, out{d.ID, d.URL, d.EventKind, d.Status, d.Attempts, d.LastStatus, d.LastError})
	169	}
	170	return c.emit(rows, func(w io.Writer) {
	171	for _, d := range rows {
	172	extra := ""
	173	if d.LastError != "" {
	174	extra = "\t" + d.LastError
	175	}
	176	fmt.Fprintf(w, "%d\t%s\t%s\t%s (%d attempts)%s\n", d.ID, d.Event, d.URL, d.Status, d.Attempts, extra)
	177	}
	178	})
	179	}
	180
	181	func runWebhookRedeliver(c *Ctx, args []string) int {
	182	if len(args) != 2 {
	183	return c.fail(protocol.ExitUsage, "usage: webhook redeliver <owner/name> <delivery-id>")
	184	}
	185	repo, code := resolveRepo(c, args[0], policy.CanAdmin)
	186	if code >= 0 {
	187	return code
	188	}
	189	id, err := strconv.ParseInt(args[1], 10, 64)
	190	if err != nil {
	191	return c.fail(protocol.ExitUsage, "bad delivery id %q", args[1])
	192	}
	193	if err := c.Store.Redeliver(repo.ID, id); err != nil {
	194	if errors.Is(err, store.ErrNotFound) {
	195	return c.fail(protocol.ExitNotFound, "no delivery %d on %s", id, repo.Path())
	196	}
	197	return c.fail(protocol.ExitFailure, "%v", err)
	198	}
	199	return c.emit(map[string]any{"requeued": id}, func(w io.Writer) {
	200	fmt.Fprintf(w, "delivery %d requeued\n", id)
	201	})
	202	}

internal/hookd/hookd.go +5

		@@ -167,6 +167,11 @@ func (s Server) preReceive(req Request, dec json.Decoder, enc *json.Encoder) {
167	167	// place a hook writes outside its own repository.
168	168	func (s *Server) postReceive(req Request) {
169	169	for _, u := range req.Updates {
	170	// Every ref update is an event webhooks can subscribe to.
	171	s.st.RecordEvent(req.RepoID, req.UserID, "push", fmt.Sprintf(
	172	`{"ref":%q,"old":%q,"new":%q,"forced":%v,"deleted":%v}`,
	173	u.Ref, u.Old, u.New, u.IsForce, u.IsDelete))
	174
170	175	branch, ok := cutHeads(u.Ref)
171	176	if !ok {
172	177	continue

internal/store/issues.go +24 −3

		@@ -220,13 +220,34 @@ func (s *Store) SetIssueAssignee(issueID, userID int64, add bool) error {
220	220	return nil
221	221	}
222	222
223		// RecordEvent appends to the event log (the forward hook CI will consume).
	223	// RecordEvent appends to the event log and enqueues a delivery for every
	224	// active webhook on the repo whose event filter matches.
224	225	func (s *Store) RecordEvent(repoID, actorID int64, kind, dataJSON string) error {
225	226	if dataJSON == "" {
226	227	dataJSON = "{}"
227	228	}
228		_, err := s.DB.Exec(
	229	tx, err := s.DB.Begin()
	230	if err != nil {
	231	return err
	232	}
	233	defer tx.Rollback()
	234	res, err := tx.Exec(
229	235	"INSERT INTO events (repo_id, actor_id, kind, data_json) VALUES (?, ?, ?, ?)",
230	236	repoID, actorID, kind, dataJSON)
231		return err
	237	if err != nil {
	238	return err
	239	}
	240	eventID, err := res.LastInsertId()
	241	if err != nil {
	242	return err
	243	}
	244	if _, err := tx.Exec(`
	245	INSERT INTO webhook_deliveries (webhook_id, event_id)
	246	SELECT id, ? FROM webhooks
	247	WHERE repo_id = ? AND active = 1
	248	AND (events = '*' OR ',' \|\| events \|\| ',' LIKE '%,' \|\| ? \|\| ',%')`,
	249	eventID, repoID, kind); err != nil {
	250	return err
	251	}
	252	return tx.Commit()
232	253	}

internal/store/migrations/0005_webhooks.down.sql added +2

		@@ -0,0 +1,2 @@
	1	DROP TABLE webhook_deliveries;
	2	DROP TABLE webhooks;

internal/store/migrations/0005_webhooks.up.sql added +25

		@@ -0,0 +1,25 @@
	1	CREATE TABLE webhooks (
	2	id INTEGER PRIMARY KEY,
	3	repo_id INTEGER NOT NULL REFERENCES repos(id) ON DELETE CASCADE,
	4	url TEXT NOT NULL,
	5	secret TEXT NOT NULL DEFAULT '',
	6	events TEXT NOT NULL DEFAULT '*',
	7	active INTEGER NOT NULL DEFAULT 1,
	8	created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
	9	);
	10	CREATE INDEX webhooks_repo ON webhooks(repo_id);
	11
	12	CREATE TABLE webhook_deliveries (
	13	id INTEGER PRIMARY KEY,
	14	webhook_id INTEGER NOT NULL REFERENCES webhooks(id) ON DELETE CASCADE,
	15	event_id INTEGER NOT NULL REFERENCES events(id) ON DELETE CASCADE,
	16	attempts INTEGER NOT NULL DEFAULT 0,
	17	next_attempt_at TEXT,
	18	delivered_at TEXT,
	19	failed_at TEXT,
	20	last_status INTEGER,
	21	last_error TEXT,
	22	created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
	23	);
	24	CREATE INDEX webhook_deliveries_due ON webhook_deliveries(next_attempt_at)
	25	WHERE delivered_at IS NULL AND failed_at IS NULL;

internal/store/webhooks.go added +174

		@@ -0,0 +1,174 @@
	1	package store
	2
	3	import (
	4	"time"
	5	)
	6
	7	type Webhook struct {
	8	ID int64
	9	URL string
	10	Secret string
	11	Events string // "*" or comma-separated kinds
	12	Active bool
	13	CreatedAt string
	14	}
	15
	16	type Delivery struct {
	17	ID int64
	18	WebhookID int64
	19	URL string
	20	Secret string
	21	EventID int64
	22	EventKind string
	23	RepoPath string
	24	Actor string
	25	DataJSON string
	26	EventAt string
	27	Attempts int
	28	}
	29
	30	type DeliveryStatus struct {
	31	ID int64
	32	URL string
	33	EventKind string
	34	Status string // pending \| delivered \| failed
	35	Attempts int
	36	LastStatus int
	37	LastError string
	38	CreatedAt string
	39	}
	40
	41	func (s *Store) AddWebhook(repoID int64, url, secret, events string) (int64, error) {
	42	res, err := s.DB.Exec(
	43	"INSERT INTO webhooks (repo_id, url, secret, events) VALUES (?, ?, ?, ?)",
	44	repoID, url, secret, events)
	45	if err != nil {
	46	return 0, err
	47	}
	48	return res.LastInsertId()
	49	}
	50
	51	func (s *Store) ListWebhooks(repoID int64) ([]Webhook, error) {
	52	rows, err := s.DB.Query(
	53	"SELECT id, url, secret, events, active, created_at FROM webhooks WHERE repo_id = ? ORDER BY id", repoID)
	54	if err != nil {
	55	return nil, err
	56	}
	57	defer rows.Close()
	58	var out []Webhook
	59	for rows.Next() {
	60	var w Webhook
	61	var active int
	62	if err := rows.Scan(&w.ID, &w.URL, &w.Secret, &w.Events, &active, &w.CreatedAt); err != nil {
	63	return nil, err
	64	}
	65	w.Active = active != 0
	66	out = append(out, w)
	67	}
	68	return out, rows.Err()
	69	}
	70
	71	func (s *Store) RemoveWebhook(repoID, hookID int64) error {
	72	res, err := s.DB.Exec("DELETE FROM webhooks WHERE repo_id = ? AND id = ?", repoID, hookID)
	73	if err != nil {
	74	return err
	75	}
	76	if n, _ := res.RowsAffected(); n == 0 {
	77	return ErrNotFound
	78	}
	79	return nil
	80	}
	81
	82	// DueDeliveries returns pending deliveries whose time has come, with the
	83	// event and hook context needed to send them.
	84	func (s *Store) DueDeliveries(limit int) ([]Delivery, error) {
	85	rows, err := s.DB.Query(`
	86	SELECT d.id, d.webhook_id, w.url, w.secret, d.event_id, e.kind,
	87	COALESCE(u2.username, o.name, '') \|\| '/' \|\| COALESCE(r.name, ''),
	88	COALESCE(u.username, ''), e.data_json, e.created_at, d.attempts
	89	FROM webhook_deliveries d
	90	JOIN webhooks w ON w.id = d.webhook_id
	91	JOIN events e ON e.id = d.event_id
	92	LEFT JOIN users u ON u.id = e.actor_id
	93	LEFT JOIN repos r ON r.id = e.repo_id
	94	LEFT JOIN users u2 ON r.owner_kind = 'user' AND u2.id = r.owner_id
	95	LEFT JOIN orgs o ON r.owner_kind = 'org' AND o.id = r.owner_id
	96	WHERE d.delivered_at IS NULL AND d.failed_at IS NULL
	97	AND (d.next_attempt_at IS NULL OR d.next_attempt_at <= ?)
	98	ORDER BY d.id LIMIT ?`, fmtTime(time.Now()), limit)
	99	if err != nil {
	100	return nil, err
	101	}
	102	defer rows.Close()
	103	var out []Delivery
	104	for rows.Next() {
	105	var d Delivery
	106	if err := rows.Scan(&d.ID, &d.WebhookID, &d.URL, &d.Secret, &d.EventID, &d.EventKind,
	107	&d.RepoPath, &d.Actor, &d.DataJSON, &d.EventAt, &d.Attempts); err != nil {
	108	return nil, err
	109	}
	110	out = append(out, d)
	111	}
	112	return out, rows.Err()
	113	}
	114
	115	func (s *Store) MarkDelivered(id int64, status int) error {
	116	_, err := s.DB.Exec(`
	117	UPDATE webhook_deliveries SET delivered_at = strftime('%Y-%m-%dT%H:%M:%fZ','now'),
	118	attempts = attempts + 1, last_status = ?, last_error = NULL WHERE id = ?`, status, id)
	119	return err
	120	}
	121
	122	// MarkAttemptFailed records a failed attempt; nextAt nil dead-letters it.
	123	func (s Store) MarkAttemptFailed(id int64, status int, errMsg string, nextAt time.Time) error {
	124	if nextAt == nil {
	125	_, err := s.DB.Exec(`
	126	UPDATE webhook_deliveries SET failed_at = strftime('%Y-%m-%dT%H:%M:%fZ','now'),
	127	attempts = attempts + 1, last_status = ?, last_error = ? WHERE id = ?`, status, errMsg, id)
	128	return err
	129	}
	130	_, err := s.DB.Exec(`
	131	UPDATE webhook_deliveries SET attempts = attempts + 1, last_status = ?, last_error = ?,
	132	next_attempt_at = ? WHERE id = ?`, status, errMsg, fmtTime(*nextAt), id)
	133	return err
	134	}
	135
	136	func (s *Store) ListDeliveries(repoID int64, limit int) ([]DeliveryStatus, error) {
	137	rows, err := s.DB.Query(`
	138	SELECT d.id, w.url, e.kind,
	139	CASE WHEN d.delivered_at IS NOT NULL THEN 'delivered'
	140	WHEN d.failed_at IS NOT NULL THEN 'failed'
	141	ELSE 'pending' END,
	142	d.attempts, COALESCE(d.last_status, 0), COALESCE(d.last_error, ''), d.created_at
	143	FROM webhook_deliveries d
	144	JOIN webhooks w ON w.id = d.webhook_id
	145	JOIN events e ON e.id = d.event_id
	146	WHERE w.repo_id = ? ORDER BY d.id DESC LIMIT ?`, repoID, limit)
	147	if err != nil {
	148	return nil, err
	149	}
	150	defer rows.Close()
	151	var out []DeliveryStatus
	152	for rows.Next() {
	153	var d DeliveryStatus
	154	if err := rows.Scan(&d.ID, &d.URL, &d.EventKind, &d.Status, &d.Attempts, &d.LastStatus, &d.LastError, &d.CreatedAt); err != nil {
	155	return nil, err
	156	}
	157	out = append(out, d)
	158	}
	159	return out, rows.Err()
	160	}
	161
	162	// Redeliver resets a delivery for an immediate retry.
	163	func (s *Store) Redeliver(repoID, deliveryID int64) error {
	164	res, err := s.DB.Exec(`
	165	UPDATE webhook_deliveries SET delivered_at = NULL, failed_at = NULL, next_attempt_at = NULL
	166	WHERE id = ? AND webhook_id IN (SELECT id FROM webhooks WHERE repo_id = ?)`, deliveryID, repoID)
	167	if err != nil {
	168	return err
	169	}
	170	if n, _ := res.RowsAffected(); n == 0 {
	171	return ErrNotFound
	172	}
	173	return nil
	174	}

internal/webhook/webhook.go added +182

		@@ -0,0 +1,182 @@
	1	// Package webhook delivers events to registered endpoints: HMAC-signed
	2	// JSON POSTs with bounded retries, exponential backoff, and dead-lettering.
	3	package webhook
	4
	5	import (
	6	"bytes"
	7	"context"
	8	"crypto/hmac"
	9	"crypto/sha256"
	10	"encoding/hex"
	11	"encoding/json"
	12	"fmt"
	13	"io"
	14	"log/slog"
	15	"net"
	16	"net/http"
	17	"net/url"
	18	"time"
	19
	20	"gitbay.org/gitbay/internal/store"
	21	)
	22
	23	// ValidateURL rejects URLs a webhook must not target: non-HTTP schemes and,
	24	// unless allowLocal, anything resolving to loopback, private, or link-local
	25	// addresses (SSRF).
	26	func ValidateURL(raw string, allowLocal bool) error {
	27	u, err := url.Parse(raw)
	28	if err != nil {
	29	return fmt.Errorf("invalid URL: %w", err)
	30	}
	31	if u.Scheme != "http" && u.Scheme != "https" {
	32	return fmt.Errorf("webhook URLs must be http or https")
	33	}
	34	if u.Hostname() == "" {
	35	return fmt.Errorf("webhook URL has no host")
	36	}
	37	if allowLocal {
	38	return nil
	39	}
	40	ips, err := net.LookupIP(u.Hostname())
	41	if err != nil {
	42	return fmt.Errorf("cannot resolve %s: %w", u.Hostname(), err)
	43	}
	44	for _, ip := range ips {
	45	if isForbidden(ip) {
	46	return fmt.Errorf("webhook target %s resolves to a private or local address; refusing (SSRF)", u.Hostname())
	47	}
	48	}
	49	return nil
	50	}
	51
	52	func isForbidden(ip net.IP) bool {
	53	return ip.IsLoopback() \|\| ip.IsPrivate() \|\| ip.IsLinkLocalUnicast() \|\|
	54	ip.IsLinkLocalMulticast() \|\| ip.IsUnspecified()
	55	}
	56
	57	type Deliverer struct {
	58	St *store.Store
	59	AllowLocal bool
	60	RetryBase time.Duration // first retry delay; doubles per attempt
	61	MaxAttempts int
	62	client *http.Client
	63	}
	64
	65	// New builds a deliverer whose dialer re-checks resolved addresses at
	66	// connect time, so a DNS answer that changes after ValidateURL still cannot
	67	// reach private space.
	68	func New(st store.Store, allowLocal bool, retryBase time.Duration) Deliverer {
	69	d := &Deliverer{St: st, AllowLocal: allowLocal, RetryBase: retryBase, MaxAttempts: 5}
	70	dialer := &net.Dialer{Timeout: 5 * time.Second}
	71	d.client = &http.Client{
	72	Timeout: 10 * time.Second,
	73	CheckRedirect: func(http.Request, []http.Request) error {
	74	return http.ErrUseLastResponse // never follow redirects
	75	},
	76	Transport: &http.Transport{
	77	DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
	78	host, port, err := net.SplitHostPort(addr)
	79	if err != nil {
	80	return nil, err
	81	}
	82	ips, err := net.DefaultResolver.LookupIP(ctx, "ip", host)
	83	if err != nil {
	84	return nil, err
	85	}
	86	for _, ip := range ips {
	87	if !allowLocal && isForbidden(ip) {
	88	return nil, fmt.Errorf("refusing connection to private address %s", ip)
	89	}
	90	}
	91	return dialer.DialContext(ctx, network, net.JoinHostPort(ips[0].String(), port))
	92	},
	93	},
	94	}
	95	return d
	96	}
	97
	98	// Run polls for due deliveries until ctx is done.
	99	func (d *Deliverer) Run(ctx context.Context) {
	100	tick := time.NewTicker(2 * time.Second)
	101	defer tick.Stop()
	102	for {
	103	select {
	104	case <-ctx.Done():
	105	return
	106	case <-tick.C:
	107	due, err := d.St.DueDeliveries(20)
	108	if err != nil {
	109	slog.Error("webhook: listing due deliveries", "err", err)
	110	continue
	111	}
	112	for _, dl := range due {
	113	d.deliver(ctx, dl)
	114	}
	115	}
	116	}
	117	}
	118
	119	type payload struct {
	120	Event string `json:"event"`
	121	Repo string `json:"repo"`
	122	Actor string `json:"actor,omitempty"`
	123	CreatedAt string `json:"created_at"`
	124	Data json.RawMessage `json:"data"`
	125	}
	126
	127	func (d *Deliverer) deliver(ctx context.Context, dl store.Delivery) {
	128	body, err := json.Marshal(payload{
	129	Event: dl.EventKind, Repo: dl.RepoPath, Actor: dl.Actor,
	130	CreatedAt: dl.EventAt, Data: json.RawMessage(dl.DataJSON),
	131	})
	132	if err != nil {
	133	d.fail(dl, 0, "marshal: "+err.Error())
	134	return
	135	}
	136	req, err := http.NewRequestWithContext(ctx, "POST", dl.URL, bytes.NewReader(body))
	137	if err != nil {
	138	d.fail(dl, 0, "request: "+err.Error())
	139	return
	140	}
	141	req.Header.Set("Content-Type", "application/json")
	142	req.Header.Set("User-Agent", "gitbay-webhook")
	143	req.Header.Set("X-Gitbay-Event", dl.EventKind)
	144	req.Header.Set("X-Gitbay-Delivery", fmt.Sprint(dl.ID))
	145	if dl.Secret != "" {
	146	mac := hmac.New(sha256.New, []byte(dl.Secret))
	147	mac.Write(body)
	148	req.Header.Set("X-Gitbay-Signature-256", "sha256="+hex.EncodeToString(mac.Sum(nil)))
	149	}
	150
	151	resp, err := d.client.Do(req)
	152	if err != nil {
	153	d.fail(dl, 0, err.Error())
	154	return
	155	}
	156	io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
	157	resp.Body.Close()
	158	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
	159	if err := d.St.MarkDelivered(dl.ID, resp.StatusCode); err != nil {
	160	slog.Error("webhook: marking delivered", "err", err)
	161	}
	162	return
	163	}
	164	d.fail(dl, resp.StatusCode, fmt.Sprintf("endpoint returned %d", resp.StatusCode))
	165	}
	166
	167	// fail schedules a retry with exponential backoff, dead-lettering after
	168	// MaxAttempts.
	169	func (d *Deliverer) fail(dl store.Delivery, status int, msg string) {
	170	attempt := dl.Attempts + 1 // the one that just happened
	171	if attempt >= d.MaxAttempts {
	172	if err := d.St.MarkAttemptFailed(dl.ID, status, msg, nil); err != nil {
	173	slog.Error("webhook: dead-lettering", "err", err)
	174	}
	175	slog.Warn("webhook dead-lettered", "delivery", dl.ID, "url", dl.URL, "err", msg)
	176	return
	177	}
	178	next := time.Now().Add(d.RetryBase << (attempt - 1))
	179	if err := d.St.MarkAttemptFailed(dl.ID, status, msg, &next); err != nil {
	180	slog.Error("webhook: scheduling retry", "err", err)
	181	}
	182	}

Commit 9cdfb3a89d

Commit `9cdfb3a89d`