Commit `e0b9ff1afa`

e0b9ff1afa876e443557111435101a3394175bdf

parent: cd8af93e07

Verified · cmc ci/build: success ci/sonar: success ci/test: success ci/vuln: success

cmc <hello@cleberg.net> · 2026-09-05 04:58 UTC

ci: path filters so a job runs only when its paths changed

A job in .gitbay/ci.yml gains paths and paths-ignore globs, matched
against the files a push changed. A trailing /** matches a directory
at any depth, since path.Match's * does not cross a separator. A job
runs when Paths is empty or a changed file matches one of its
patterns, and is held back only when PathsIgnore is non-empty and
every changed file matches one of those.

The filter fails open: a new branch with no diff base, a failed diff,
or a job declaring neither key all run the job rather than skip it
silently. QueueBranchBuilds and queueJobs take the pre-push sha to
diff against; QueueMRBuilds passes an empty one, unaffected. The
changed-file list is computed once per push, only when some job
actually declares a filter.

Closes #169

Layout: unified · split

e2e/cipaths_test.go added +64

		@@ -0,0 +1,64 @@
	1	package e2e
	2
	3	import (
	4	"os"
	5	"path/filepath"
	6	"strings"
	7	"testing"
	8	)
	9
	10	// A job with paths only builds when the push touched something it names.
	11	// A doc-only push queues nothing; a push touching the named path queues
	12	// the job, same as before path filters existed.
	13	func TestCIPaths(t *testing.T) {
	14	inst := startInstance(t)
	15	aliceKey := inst.newKey(t, "alice")
	16	inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
	17
	18	if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
	19	t.Fatalf("repo create: %s", errOut)
	20	}
	21	work := t.TempDir()
	22	env := inst.gitEnv(aliceKey)
	23	mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
	24	dir := filepath.Join(work, "w")
	25	os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
	26	os.MkdirAll(filepath.Join(dir, "src"), 0o755)
	27	os.MkdirAll(filepath.Join(dir, "docs"), 0o755)
	28	os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte(
	29	"jobs:\n unit:\n paths:\n - src/**\n steps:\n - echo fine\n"), 0o644)
	30	os.WriteFile(filepath.Join(dir, "src", "x.go"), []byte("package x\n"), 0o644)
	31	os.WriteFile(filepath.Join(dir, "docs", "x.md"), []byte("# x\n"), 0o644)
	32	mustGit(t, dir, env, "checkout", "-q", "-b", "main")
	33	mustGit(t, dir, env, "add", ".")
	34	mustGit(t, dir, env, "commit", "-q", "-m", "base")
	35	mustGit(t, dir, env, "push", "-q", "origin", "main")
	36	// A new branch has no diff base, so the filter fails open: the base
	37	// push above already queued build 1.
	38	before := strings.Count(inst.buildList(t, aliceKey), "\n")
	39	if before != 1 {
	40	t.Fatalf("base push did not queue exactly one build:\n%s", inst.buildList(t, aliceKey))
	41	}
	42
	43	// A push touching only docs does not match src/**: no build queued.
	44	os.WriteFile(filepath.Join(dir, "docs", "x.md"), []byte("# x changed\n"), 0o644)
	45	mustGit(t, dir, env, "add", ".")
	46	mustGit(t, dir, env, "commit", "-q", "-m", "docs only")
	47	mustGit(t, dir, env, "push", "-q", "origin", "main")
	48	if out := inst.buildList(t, aliceKey); strings.Count(out, "\n") != before {
	49	t.Fatalf("doc-only push queued a build:\n%s", out)
	50	}
	51
	52	// A push touching src matches: a build is queued.
	53	os.WriteFile(filepath.Join(dir, "src", "x.go"), []byte("package x\n\nvar y int\n"), 0o644)
	54	mustGit(t, dir, env, "add", ".")
	55	mustGit(t, dir, env, "commit", "-q", "-m", "src change")
	56	mustGit(t, dir, env, "push", "-q", "origin", "main")
	57	out := inst.buildList(t, aliceKey)
	58	if strings.Count(out, "\n") != before+1 {
	59	t.Fatalf("src push did not queue a build:\n%s", out)
	60	}
	61	if !strings.Contains(out, "unit\tpending") {
	62	t.Fatalf("src push did not queue the unit job:\n%s", out)
	63	}
	64	}

internal/ci/ci.go +32 −8

		@@ -25,15 +25,18 @@ const (
25	25	maxJobs = 10
26	26	maxSteps = 50
27	27	maxStepSize = 4096
	28	maxPaths = 50
28	29	)
29	30
30	31	var jobName = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{0,39}$`)
31	32
32	33	type Job struct {
33		Name string
34		Steps []string
35		Schedule string // cron expression; scheduled jobs run on schedule, not on push
36		Tags string // tag glob (e.g. "v*"); tag jobs run on matching tag pushes only
	34	Name string
	35	Steps []string
	36	Schedule string // cron expression; scheduled jobs run on schedule, not on push
	37	Tags string // tag glob (e.g. "v*"); tag jobs run on matching tag pushes only
	38	Paths []string // globs; the job runs only when a changed file matches one
	39	PathsIgnore []string // globs; the job is skipped when every changed file matches one
37	40	}
38	41
39	42	// Parse returns the jobs in name order, or an error describing the first
		@@ -41,9 +44,11 @@ type Job struct {
41	44	func Parse(raw []byte) ([]Job, error) {
42	45	var doc struct {
43	46	Jobs map[string]struct {
44		Steps []string `yaml:"steps"`
45		Schedule string `yaml:"schedule"`
46		Tags string `yaml:"tags"`
	47	Steps []string `yaml:"steps"`
	48	Schedule string `yaml:"schedule"`
	49	Tags string `yaml:"tags"`
	50	Paths []string `yaml:"paths"`
	51	PathsIgnore []string `yaml:"paths-ignore"`
47	52	} `yaml:"jobs"`
48	53	}
49	54	if err := yaml.Unmarshal(raw, &doc); err != nil {
		@@ -84,7 +89,26 @@ func Parse(raw []byte) ([]Job, error) {
84	89	return nil, fmt.Errorf("job %q: schedule and tags are mutually exclusive", name)
85	90	}
86	91	}
87		jobs = append(jobs, Job{Name: name, Steps: j.Steps, Schedule: j.Schedule, Tags: j.Tags})
	92	if len(j.Paths) > maxPaths {
	93	return nil, fmt.Errorf("job %q has %d path patterns; max %d", name, len(j.Paths), maxPaths)
	94	}
	95	for _, p := range j.Paths {
	96	if _, err := path.Match(p, "x"); err != nil {
	97	return nil, fmt.Errorf("job %q: bad path pattern %q", name, p)
	98	}
	99	}
	100	if len(j.PathsIgnore) > maxPaths {
	101	return nil, fmt.Errorf("job %q has %d paths-ignore patterns; max %d", name, len(j.PathsIgnore), maxPaths)
	102	}
	103	for _, p := range j.PathsIgnore {
	104	if _, err := path.Match(p, "x"); err != nil {
	105	return nil, fmt.Errorf("job %q: bad paths-ignore pattern %q", name, p)
	106	}
	107	}
	108	jobs = append(jobs, Job{
	109	Name: name, Steps: j.Steps, Schedule: j.Schedule, Tags: j.Tags,
	110	Paths: j.Paths, PathsIgnore: j.PathsIgnore,
	111	})
88	112	}
89	113	sort.Slice(jobs, func(i, k int) bool { return jobs[i].Name < jobs[k].Name })
90	114	return jobs, nil

internal/ci/paths.go added +67

		@@ -0,0 +1,67 @@
	1	package ci
	2
	3	import (
	4	"path"
	5	"strings"
	6	)
	7
	8	// zeroSHA is git's null object id: the old side of a ref update that
	9	// created the ref, carrying no diff base.
	10	const zeroSHA = "0000000000000000000000000000000000000000"
	11
	12	// Match reports whether a changed file path matches a job's glob.
	13	// A trailing /** matches the directory and everything below it at any
	14	// depth; anything else goes to path.Match, whose * stops at a separator.
	15	func Match(pattern, file string) bool {
	16	if prefix, ok := strings.CutSuffix(pattern, "/**"); ok {
	17	return file == prefix \|\| strings.HasPrefix(file, prefix+"/")
	18	}
	19	ok, err := path.Match(pattern, file)
	20	return err == nil && ok
	21	}
	22
	23	// Selected reports whether a job's path filters admit a push that
	24	// changed the given files. Call it only once the changed-file list is
	25	// known; a caller that cannot compute one must run the job instead of
	26	// calling this.
	27	//
	28	// The job runs when Paths is empty or at least one file matches one of
	29	// its patterns, and is then held back only if PathsIgnore is non-empty
	30	// and every file matches one of its patterns.
	31	func Selected(j Job, files []string) bool {
	32	if len(j.Paths) > 0 {
	33	hit := false
	34	for _, f := range files {
	35	for _, p := range j.Paths {
	36	if Match(p, f) {
	37	hit = true
	38	}
	39	}
	40	}
	41	if !hit {
	42	return false
	43	}
	44	}
	45	if len(j.PathsIgnore) > 0 {
	46	for _, f := range files {
	47	ignored := false
	48	for _, p := range j.PathsIgnore {
	49	if Match(p, f) {
	50	ignored = true
	51	}
	52	}
	53	if !ignored {
	54	return true
	55	}
	56	}
	57	return false
	58	}
	59	return true
	60	}
	61
	62	// HasDiffBase reports whether old names a commit a diff can start from.
	63	// A branch's first push carries an empty or all-zero old sha, so there
	64	// is nothing to compare the new commit against.
	65	func HasDiffBase(old string) bool {
	66	return old != "" && old != zeroSHA
	67	}

internal/ci/paths_test.go added +79

		@@ -0,0 +1,79 @@
	1	package ci
	2
	3	import "testing"
	4
	5	func TestMatch(t *testing.T) {
	6	cases := []struct {
	7	pattern, file string
	8	want bool
	9	}{
	10	{".gitbay/wiki/**", ".gitbay/wiki/Home.md", true},
	11	{".gitbay/wiki/**", ".gitbay/wiki/sub/Page.md", true}, // nested: the case a reader assumes works
	12	{".gitbay/wiki/**", ".gitbay/wiki", true},
	13	{".gitbay/wiki/**", "other/Home.md", false},
	14	{"*.md", "README.md", true},
	15	{"*.md", "docs/README.md", false},
	16	{"docs/*.md", "docs/a.md", true},
	17	{"docs/*.md", "docs/sub/a.md", false},
	18	}
	19	for _, c := range cases {
	20	if got := Match(c.pattern, c.file); got != c.want {
	21	t.Errorf("Match(%q, %q) = %v, want %v", c.pattern, c.file, got, c.want)
	22	}
	23	}
	24	}
	25
	26	func TestSelected(t *testing.T) {
	27	cases := []struct {
	28	name string
	29	job Job
	30	files []string
	31	want bool
	32	}{
	33	{"paths hit", Job{Paths: []string{"src/**"}}, []string{"src/x.go"}, true},
	34	{"paths miss", Job{Paths: []string{"src/**"}}, []string{"docs/x.md"}, false},
	35	{"paths-ignore covers every file", Job{PathsIgnore: []string{"docs/**"}},
	36	[]string{"docs/a.md", "docs/b.md"}, false},
	37	{"paths-ignore covers some files", Job{PathsIgnore: []string{"docs/**"}},
	38	[]string{"docs/a.md", "src/x.go"}, true},
	39	{"neither key", Job{}, []string{"anything.txt"}, true},
	40	}
	41	for _, c := range cases {
	42	if got := Selected(c.job, c.files); got != c.want {
	43	t.Errorf("%s: Selected() = %v, want %v", c.name, got, c.want)
	44	}
	45	}
	46	}
	47
	48	func TestHasDiffBase(t *testing.T) {
	49	cases := []struct {
	50	old string
	51	want bool
	52	}{
	53	{"", false},
	54	{"0000000000000000000000000000000000000000", false},
	55	{"a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", true},
	56	}
	57	for _, c := range cases {
	58	if got := HasDiffBase(c.old); got != c.want {
	59	t.Errorf("HasDiffBase(%q) = %v, want %v", c.old, got, c.want)
	60	}
	61	}
	62	}
	63
	64	func TestParsePaths(t *testing.T) {
	65	raw := []byte("jobs:\n unit:\n steps:\n - echo hi\n paths:\n - src/\n paths-ignore:\n - docs/\n")
	66	jobs, err := Parse(raw)
	67	if err != nil {
	68	t.Fatalf("Parse: %v", err)
	69	}
	70	if len(jobs) != 1 \|\| len(jobs[0].Paths) != 1 \|\| jobs[0].Paths[0] != "src/**" \|\|
	71	len(jobs[0].PathsIgnore) != 1 \|\| jobs[0].PathsIgnore[0] != "docs/**" {
	72	t.Fatalf("Parse did not round-trip paths: %+v", jobs)
	73	}
	74
	75	bad := []byte("jobs:\n unit:\n steps:\n - echo hi\n paths:\n - \"[\"\n")
	76	if _, err := Parse(bad); err == nil {
	77	t.Fatal("Parse accepted a malformed path pattern")
	78	}
	79	}

internal/control/build.go +30 −5

		@@ -500,12 +500,14 @@ func runRunnerDone(c *Ctx, args []string) int {
500	500	//
501	501	// Both paths that move a branch call this: post-receive for a push, and
502	502	// the merge path for a merge, which updates the ref directly and so never
503		// reaches a hook.
	503	// reaches a hook. old is the branch's sha before this update, the diff
	504	// base a job's path filters run against; an empty old, from a new
	505	// branch, cannot be diffed and runs every job.
504	506	func QueueBranchBuilds(
505	507	st *store.Store, root, siteURL string,
506		repo store.Repo, userID int64, branch, sha string, now time.Time,
	508	repo store.Repo, userID int64, branch, old, sha string, now time.Time,
507	509	) {
508		queueJobs(st, root, siteURL, repo, userID, branch, sha, now, true, branch == repo.DefaultBranch)
	510	queueJobs(st, root, siteURL, repo, userID, branch, old, sha, now, true, branch == repo.DefaultBranch)
509	511	}
510	512
511	513	// QueueMRBuilds queues the push jobs for a merge request head fetched
		@@ -519,12 +521,14 @@ func QueueMRBuilds(
519	521	st *store.Store, root, siteURL string,
520	522	repo store.Repo, userID, n int64, sha string,
521	523	) {
522		queueJobs(st, root, siteURL, repo, userID, mrHeadRef(n), sha, time.Now(), false, false)
	524	// No old sha: fails open and runs every job, matching today's
	525	// behaviour for a merge request head.
	526	queueJobs(st, root, siteURL, repo, userID, mrHeadRef(n), "", sha, time.Now(), false, false)
523	527	}
524	528
525	529	func queueJobs(
526	530	st *store.Store, root, siteURL string,
527		repo store.Repo, userID int64, ref, sha string, now time.Time,
	531	repo store.Repo, userID int64, ref, old, sha string, now time.Time,
528	532	trusted, syncSchedules bool,
529	533	) {
530	534	dir := RepoDir(root, repo.OwnerName, repo.Name)
		@@ -546,6 +550,24 @@ func queueJobs(
546	550	if err != nil {
547	551	built = nil
548	552	}
	553	// The changed-file list a job's path filters run against, computed
	554	// once and only if some job actually declares one. When the diff
	555	// base does not exist or the diff itself fails, filtered stays
	556	// false and every job runs: a filter that cannot be evaluated must
	557	// not silently skip CI.
	558	filtered := false
	559	var changed []string
	560	for _, j := range jobs {
	561	if len(j.Paths) == 0 && len(j.PathsIgnore) == 0 {
	562	continue
	563	}
	564	if ci.HasDiffBase(old) {
	565	if files, err := gitutil.DiffFiles(dir, old, sha); err == nil {
	566	changed, filtered = files, true
	567	}
	568	}
	569	break
	570	}
549	571	var schedules []store.Schedule
550	572	for _, j := range jobs {
551	573	// Tag jobs run on matching tag pushes only.
		@@ -566,6 +588,9 @@ func queueJobs(
566	588	}
567	589	continue
568	590	}
	591	if filtered && !ci.Selected(j, changed) {
	592	continue
	593	}
569	594	steps, _ := json.Marshal(j.Steps)
570	595	n, err := st.CreateBuild(repo.ID, j.Name, sha, ref, string(steps), trusted)
571	596	if err != nil {

internal/control/mr.go +1 −1

		@@ -1089,7 +1089,7 @@ func runMRMerge(c *Ctx, args []string) int {
1089	1089	`{"ref":%q,"old":%q,"new":%q,"forced":false,"deleted":false}`,
1090	1090	targetRef, targetSHA, newSHA))
1091	1091	QueueBranchBuilds(c.Store, c.Cfg.Server.Root, c.Cfg.Server.SiteURL,
1092		repo, c.User.ID, mr.TargetRef, newSHA, time.Now())
	1092	repo, c.User.ID, mr.TargetRef, targetSHA, newSHA, time.Now())
1093	1093	c.Store.MarkMirrorsDirty(repo.ID, "push")
1094	1094	if parts, err := c.Store.MRParticipants(mr.ID); err == nil {
1095	1095	notify(c, parts, notice{repo: repo, kind: "mr",

internal/hookd/hookd.go +3 −3

		@@ -214,7 +214,7 @@ func (s *Server) postReceive(req Request) {
214	214	}
215	215	// A branch push with a .gitbay/ci.yml queues one build per job.
216	216	if pushedRepoErr == nil && !u.IsDelete {
217		s.queueBuilds(pushedRepo, req.UserID, branch, u.New)
	217	s.queueBuilds(pushedRepo, req.UserID, branch, u.Old, u.New)
218	218	}
219	219	if u.IsForce {
220	220	s.st.Audit(req.UserID, "push.forced", map[string]any{
		@@ -271,10 +271,10 @@ func (s *Server) postReceive(req Request) {
271	271
272	272	// queueBuilds queues the push jobs for a branch update. The work is
273	273	// shared with the merge path, which moves a ref without reaching a hook.
274		func (s *Server) queueBuilds(repo store.Repo, userID int64, branch, sha string) {
	274	func (s *Server) queueBuilds(repo store.Repo, userID int64, branch, old, sha string) {
275	275	control.QueueBranchBuilds(
276	276	s.st, s.cfg.Server.Root, s.cfg.Server.SiteURL,
277		repo, userID, branch, sha, time.Now())
	277	repo, userID, branch, old, sha, time.Now())
278	278	}
279	279
280	280	// queueTagBuilds runs the jobs whose tag pattern matches a pushed tag.

Commit e0b9ff1afa

Commit `e0b9ff1afa`