6c97fd8119 · krz/gitbay

cmd/gitbay/main.go +5

		@@ -311,6 +311,11 @@ func repoCmd() *cobra.Command {
311	311	pass("remove", "remove a mirror: <id>", passOpts{server: []string{"repo", "mirror", "remove"}, needsRepo: true}),
312	312	pass("sync", "schedule an immediate sync", passOpts{server: []string{"repo", "mirror", "sync"}, needsRepo: true}),
313	313	),
	314	group("deps", "check dependencies against upstream registries",
	315	pass("enable", "check this repo's dependencies for updates", passOpts{server: []string{"repo", "deps", "enable"}, needsRepo: true}),
	316	pass("disable", "stop checking dependencies", passOpts{server: []string{"repo", "deps", "disable"}, needsRepo: true}),
	317	pass("status", "show check state and what is behind", passOpts{server: []string{"repo", "deps", "status"}, needsRepo: true}),
	318	),
314	319	group("secret", "build secrets (values on stdin, injected into build env)",
315	320	pass("set", "set a secret: <NAME> (value on stdin)", passOpts{server: []string{"repo", "secret", "set"}, needsRepo: true, alwaysStdin: true}),
316	321	pass("list", "list secret names", passOpts{server: []string{"repo", "secret", "list"}, needsRepo: true}),

cmd/gitbayd/main.go +5

		@@ -18,9 +18,11 @@ import (
18	18	"golang.org/x/crypto/acme/autocert"
19	19	"golang.org/x/crypto/ssh"
20	20
	21	"gitbay.org/gitbay/internal/buildinfo"
21	22	"gitbay.org/gitbay/internal/ci"
22	23	"gitbay.org/gitbay/internal/config"
23	24	"gitbay.org/gitbay/internal/control"
	25	"gitbay.org/gitbay/internal/deps"
24	26	"gitbay.org/gitbay/internal/gitd"
25	27	"gitbay.org/gitbay/internal/hookd"
26	28	"gitbay.org/gitbay/internal/httpd"
		@@ -165,6 +167,9 @@ func serveCmd() *cobra.Command {
165	167	RepoDir: func(owner, name string) string {
166	168	return control.RepoDir(cfg.Server.Root, owner, name)
167	169	}}).Run(whCtx)
	170	go deps.New(st, cfg, func(owner, name string) string {
	171	return control.RepoDir(cfg.Server.Root, owner, name)
	172	}, buildinfo.String()).Run(whCtx)
168	173
169	174	errCh := make(chan error, 3)
170	175	if cfg.SSH.Mode == "embedded" {

e2e/deps_test.go added +81

		@@ -0,0 +1,81 @@
	1	package e2e
	2
	3	import (
	4	"encoding/json"
	5	"strings"
	6	"testing"
	7	)
	8
	9	type depsStatus struct {
	10	Data struct {
	11	Enabled bool `json:"enabled"`
	12	IssueNumber int64 `json:"issue_number"`
	13	Behind []struct {
	14	Name string `json:"name"`
	15	} `json:"behind"`
	16	} `json:"data"`
	17	}
	18
	19	// Dependency checking is opt-in per repository, and only its administrators
	20	// can turn it on: the check tells a public registry what the repository
	21	// depends on.
	22	func TestDepsEnableDisable(t *testing.T) {
	23	inst := startInstance(t)
	24	aliceKey := inst.newKey(t, "alice")
	25	inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub",
	26	"--email", "alice@example.test", "--verified")
	27	bobKey := inst.newKey(t, "bob")
	28	inst.admin(t, "admin", "user", "create", "bob", "--key", bobKey+".pub",
	29	"--email", "bob@example.test", "--verified")
	30
	31	if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
	32	t.Fatalf("repo create: %s", errOut)
	33	}
	34
	35	status := func(key string) depsStatus {
	36	t.Helper()
	37	out, errOut, code := inst.ssh(t, key, "", "repo", "deps", "status", "alice/app", "--json")
	38	if code != 0 {
	39	t.Fatalf("deps status: %s", errOut)
	40	}
	41	var s depsStatus
	42	if err := json.Unmarshal([]byte(out), &s); err != nil {
	43	t.Fatalf("parsing status %q: %v", out, err)
	44	}
	45	return s
	46	}
	47
	48	if status(aliceKey).Data.Enabled {
	49	t.Error("checks are on before being enabled")
	50	}
	51
	52	// A reader can see the state but cannot change it.
	53	if _, errOut, code := inst.ssh(t, bobKey, "", "repo", "deps", "enable", "alice/app"); code == 0 {
	54	t.Error("a non-admin enabled dependency checks")
	55	} else if !strings.Contains(errOut, "denied") {
	56	t.Errorf("unexpected refusal: %s", errOut)
	57	}
	58
	59	if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "deps", "enable", "alice/app"); code != 0 {
	60	t.Fatalf("deps enable: %s", errOut)
	61	}
	62	got := status(aliceKey)
	63	if !got.Data.Enabled {
	64	t.Error("checks are off after being enabled")
	65	}
	66	if len(got.Data.Behind) != 0 {
	67	t.Errorf("behind = %v before any sweep", got.Data.Behind)
	68	}
	69
	70	// Enabling twice is not an error.
	71	if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "deps", "enable", "alice/app"); code != 0 {
	72	t.Fatalf("second deps enable: %s", errOut)
	73	}
	74
	75	if _, errOut, code := inst.ssh(t, aliceKey, "", "repo", "deps", "disable", "alice/app"); code != 0 {
	76	t.Fatalf("deps disable: %s", errOut)
	77	}
	78	if status(aliceKey).Data.Enabled {
	79	t.Error("checks are on after being disabled")
	80	}
	81	}

internal/config/config.go +9

		@@ -26,6 +26,7 @@ type Config struct {
26	26	Limits Limits `toml:"limits"`
27	27	Mail Mail `toml:"mail"`
28	28	Mirrors Mirrors `toml:"mirrors"`
	29	Deps Deps `toml:"deps"`
29	30	// GoImport maps vanity Go module paths to repositories, e.g.
30	31	// "gitbay.org/gitbay" = "krz/gitbay". Requests carrying ?go-get=1
31	32	// under a mapped path get a go-import meta tag.
		@@ -114,6 +115,13 @@ type Mirrors struct {
114	115	PullIntervalMinutes int `toml:"pull_interval_minutes"`
115	116	}
116	117
	118	// Deps configures the dependency-update sweep. It runs only for repos that
	119	// have opted in with `repo deps enable`, because checking a private repo
	120	// tells a public registry what it depends on.
	121	type Deps struct {
	122	CheckIntervalHours int `toml:"check_interval_hours"`
	123	}
	124
117	125	type Limits struct {
118	126	MaxPackBytes int64 `toml:"max_pack_bytes"`
119	127	MaxBlobBytes int64 `toml:"max_blob_bytes"`
		@@ -144,6 +152,7 @@ func Default() Config {
144	152	},
145	153	GitDaemon: GitDaemon{Port: 9418},
146	154	Mirrors: Mirrors{PullIntervalMinutes: 15},
	155	Deps: Deps{CheckIntervalHours: 24},
147	156	Limits: Limits{
148	157	MaxPackBytes: 2 << 30, // 2 GiB
149	158	MaxBlobBytes: 100 << 20,

internal/control/deps.go added +107

		@@ -0,0 +1,107 @@
	1	package control
	2
	3	import (
	4	"errors"
	5	"fmt"
	6	"io"
	7
	8	"gitbay.org/gitbay/internal/policy"
	9	"gitbay.org/gitbay/internal/protocol"
	10	"gitbay.org/gitbay/internal/store"
	11	)
	12
	13	func init() {
	14	// Dependency update checks. Opt-in per repository: the check tells a
	15	// public registry what the repository depends on, which is the owner's
	16	// disclosure to make, not the instance's.
	17	register(Command{Path: []string{"repo", "deps", "enable"},
	18	Summary: "check dependencies for updates: repo deps enable <owner/name>", Run: runDepsEnable})
	19	register(Command{Path: []string{"repo", "deps", "disable"},
	20	Summary: "stop checking dependencies: repo deps disable <owner/name>", Run: runDepsDisable})
	21	register(Command{Path: []string{"repo", "deps", "status"},
	22	Summary: "show dependency check state: repo deps status <owner/name>", ReadOnly: true, Run: runDepsStatus})
	23	}
	24
	25	func runDepsEnable(c *Ctx, args []string) int {
	26	if len(args) != 1 {
	27	return c.fail(protocol.ExitUsage, "usage: repo deps enable <owner/name>")
	28	}
	29	repo, code := resolveRepo(c, args[0], policy.CanAdmin)
	30	if code >= 0 {
	31	return code
	32	}
	33	if err := c.Store.EnableDepCheck(repo.ID); err != nil {
	34	return c.fail(protocol.ExitFailure, "%v", err)
	35	}
	36	return c.emit(map[string]any{"enabled": true}, func(w io.Writer) {
	37	fmt.Fprintf(w, "dependency checks enabled on %s\n", repo.Path())
	38	})
	39	}
	40
	41	func runDepsDisable(c *Ctx, args []string) int {
	42	if len(args) != 1 {
	43	return c.fail(protocol.ExitUsage, "usage: repo deps disable <owner/name>")
	44	}
	45	repo, code := resolveRepo(c, args[0], policy.CanAdmin)
	46	if code >= 0 {
	47	return code
	48	}
	49	if err := c.Store.DisableDepCheck(repo.ID); err != nil {
	50	return c.fail(protocol.ExitFailure, "%v", err)
	51	}
	52	return c.emit(map[string]any{"enabled": false}, func(w io.Writer) {
	53	fmt.Fprintf(w, "dependency checks disabled on %s\n", repo.Path())
	54	})
	55	}
	56
	57	func runDepsStatus(c *Ctx, args []string) int {
	58	if len(args) != 1 {
	59	return c.fail(protocol.ExitUsage, "usage: repo deps status <owner/name>")
	60	}
	61	repo, code := resolveRepo(c, args[0], policy.CanRead)
	62	if code >= 0 {
	63	return code
	64	}
	65	check, err := c.Store.DepCheckFor(repo.ID)
	66	if errors.Is(err, store.ErrNotFound) {
	67	return c.emit(map[string]any{"enabled": false}, func(w io.Writer) {
	68	fmt.Fprintf(w, "dependency checks are off for %s (repo deps enable %s)\n", repo.Path(), repo.Path())
	69	})
	70	}
	71	if err != nil {
	72	return c.fail(protocol.ExitFailure, "%v", err)
	73	}
	74	reports, err := c.Store.ReportedDeps(repo.ID)
	75	if err != nil {
	76	return c.fail(protocol.ExitFailure, "%v", err)
	77	}
	78	type behind struct {
	79	Ecosystem string `json:"ecosystem"`
	80	Name string `json:"name"`
	81	Current string `json:"current"`
	82	Latest string `json:"latest"`
	83	}
	84	out := struct {
	85	Enabled bool `json:"enabled"`
	86	LastCheck string `json:"last_check,omitempty"`
	87	LastError string `json:"last_error,omitempty"`
	88	IssueNumber int64 `json:"issue_number,omitempty"`
	89	Behind []behind `json:"behind"`
	90	}{Enabled: true, LastCheck: check.LastCheck, LastError: check.LastError,
	91	IssueNumber: check.IssueNumber, Behind: []behind{}}
	92	for _, r := range reports {
	93	out.Behind = append(out.Behind, behind{r.Ecosystem, r.Name, r.Current, r.Latest})
	94	}
	95	return c.emit(out, func(w io.Writer) {
	96	fmt.Fprintf(w, "checks on, last %s\n", orDash(check.LastCheck))
	97	if check.LastError != "" {
	98	fmt.Fprintf(w, "last error: %s\n", check.LastError)
	99	}
	100	if check.IssueNumber != 0 {
	101	fmt.Fprintf(w, "tracked in #%d\n", check.IssueNumber)
	102	}
	103	for _, b := range out.Behind {
	104	fmt.Fprintf(w, "%s\t%s\t%s\t-> %s\n", b.Ecosystem, b.Name, b.Current, b.Latest)
	105	}
	106	})
	107	}

internal/deps/cargo.go added +81

		@@ -0,0 +1,81 @@
	1	package deps
	2
	3	import "github.com/BurntSushi/toml"
	4
	5	// parseCargo takes the direct dependency names from Cargo.toml and their
	6	// resolved versions from Cargo.lock where one exists. Dependencies sourced
	7	// from a path or a git remote are skipped: crates.io has nothing to say
	8	// about them.
	9	func parseCargo(read ReadFile) []Dep {
	10	raw, err := read("Cargo.toml")
	11	if err != nil {
	12	return nil
	13	}
	14	var manifest struct {
	15	Dependencies map[string]any `toml:"dependencies"`
	16	DevDependencies map[string]any `toml:"dev-dependencies"`
	17	BuildDependencies map[string]any `toml:"build-dependencies"`
	18	}
	19	if toml.Unmarshal(raw, &manifest) != nil {
	20	return nil
	21	}
	22	locked := cargoLock(read)
	23	var out []Dep
	24	for _, set := range []map[string]any{manifest.Dependencies, manifest.DevDependencies, manifest.BuildDependencies} {
	25	for name, spec := range set {
	26	req, ok := cargoRequirement(spec)
	27	if !ok {
	28	continue
	29	}
	30	current := locked[name]
	31	if current == "" {
	32	current = pin(req)
	33	}
	34	if current != "" {
	35	out = append(out, Dep{Ecosystem: EcoCargo, Name: name, Current: current})
	36	}
	37	}
	38	}
	39	return out
	40	}
	41
	42	// cargoRequirement reads the version out of either dependency form —
	43	// `serde = "1.0"` or `serde = { version = "1.0", features = [...] }` —
	44	// and rejects the ones that name a source other than the registry.
	45	func cargoRequirement(spec any) (string, bool) {
	46	switch v := spec.(type) {
	47	case string:
	48	return v, true
	49	case map[string]any:
	50	if v["path"] != nil \|\| v["git"] != nil {
	51	return "", false
	52	}
	53	s, ok := v["version"].(string)
	54	return s, ok
	55	}
	56	return "", false
	57	}
	58
	59	func cargoLock(read ReadFile) map[string]string {
	60	raw, err := read("Cargo.lock")
	61	if err != nil {
	62	return nil
	63	}
	64	var lock struct {
	65	Package []struct {
	66	Name string `toml:"name"`
	67	Version string `toml:"version"`
	68	Source string `toml:"source"`
	69	} `toml:"package"`
	70	}
	71	if toml.Unmarshal(raw, &lock) != nil {
	72	return nil
	73	}
	74	out := map[string]string{}
	75	for _, p := range lock.Package {
	76	if v := pin(p.Version); v != "" {
	77	out[p.Name] = v
	78	}
	79	}
	80	return out
	81	}

internal/deps/gomod.go added +60

		@@ -0,0 +1,60 @@
	1	package deps
	2
	3	import "strings"
	4
	5	// parseGoMod reads the direct requirements from go.mod. Indirect
	6	// requirements are the module graph's business rather than the maintainer's,
	7	// and a replaced module does not come from the proxy at all, so both are
	8	// skipped.
	9	func parseGoMod(read ReadFile) []Dep {
	10	raw, err := read("go.mod")
	11	if err != nil {
	12	return nil
	13	}
	14	replaced := map[string]bool{}
	15	var reqs [][2]string
	16	block := "" // the directive whose ( ... ) block we are inside
	17	for _, line := range strings.Split(string(raw), "\n") {
	18	line = strings.TrimSpace(line)
	19	indirect := false
	20	if i := strings.Index(line, "//"); i >= 0 {
	21	indirect = strings.Contains(line[i:], "indirect")
	22	line = strings.TrimSpace(line[:i])
	23	}
	24	if line == "" {
	25	continue
	26	}
	27	directive := block
	28	if block == "" {
	29	d, rest, ok := strings.Cut(line, " ")
	30	if !ok \|\| (d != "require" && d != "replace") {
	31	continue
	32	}
	33	if rest = strings.TrimSpace(rest); rest == "(" {
	34	block = d
	35	continue
	36	}
	37	directive, line = d, rest
	38	} else if line == ")" {
	39	block = ""
	40	continue
	41	}
	42	fields := strings.Fields(line)
	43	switch {
	44	case directive == "require" && !indirect && len(fields) >= 2:
	45	reqs = append(reqs, [2]string{fields[0], fields[1]})
	46	case directive == "replace" && len(fields) >= 1:
	47	replaced[fields[0]] = true
	48	}
	49	}
	50	var out []Dep
	51	for _, r := range reqs {
	52	if replaced[r[0]] {
	53	continue
	54	}
	55	if v := pin(r[1]); v != "" {
	56	out = append(out, Dep{Ecosystem: EcoGo, Name: r[0], Current: r[1]})
	57	}
	58	}
	59	return out
	60	}

internal/deps/manifest.go added +81

		@@ -0,0 +1,81 @@
	1	package deps
	2
	3	import (
	4	"regexp"
	5	"sort"
	6	"strings"
	7	)
	8
	9	// Ecosystems, in the order they are reported.
	10	const (
	11	EcoGo = "go"
	12	EcoNPM = "npm"
	13	EcoCargo = "cargo"
	14	EcoPyPI = "pypi"
	15	)
	16
	17	// Dep is one direct dependency read from a manifest.
	18	type Dep struct {
	19	Ecosystem string
	20	Name string
	21	Current string
	22	}
	23
	24	// ReadFile returns a file from the tree being scanned, or an error when it
	25	// is absent. Manifests are read from the repository root only; a monorepo
	26	// with manifests in subdirectories is not scanned.
	27	type ReadFile func(path string) ([]byte, error)
	28
	29	// MaxDeps bounds the work one repository can create for a sweep.
	30	const MaxDeps = 300
	31
	32	// Scan returns the direct dependencies of every ecosystem whose manifest is
	33	// present, capped at MaxDeps. Dependencies whose version cannot be pinned to
	34	// an exact release — a range, a git or path source, a workspace member — are
	35	// left out: there is nothing meaningful to compare them against.
	36	func Scan(read ReadFile) []Dep {
	37	var out []Dep
	38	for _, parse := range []func(ReadFile) []Dep{parseGoMod, parseNPM, parseCargo, parsePython} {
	39	out = append(out, parse(read)...)
	40	}
	41	sort.Slice(out, func(i, j int) bool {
	42	if out[i].Ecosystem != out[j].Ecosystem {
	43	return out[i].Ecosystem < out[j].Ecosystem
	44	}
	45	return out[i].Name < out[j].Name
	46	})
	47	if len(out) > MaxDeps {
	48	out = out[:MaxDeps]
	49	}
	50	return out
	51	}
	52
	53	// exactVersion matches a release we can compare: dot-separated numbers with
	54	// an optional suffix, no range operators left in it. wildcard catches the
	55	// ranges that survive that shape — "1.x", "2.*".
	56	var (
	57	exactVersion = regexp.MustCompile(`^[0-9]+(\.[0-9]+)([.\-+_a-zA-Z0-9])$`)
	58	wildcard = regexp.MustCompile(`(^\|\.)[xX*](\.\|$)`)
	59	)
	60
	61	// pin reduces a version requirement to the single release it names, or ""
	62	// when it names a set rather than a release. A caret or tilde range pins its
	63	// floor, which is the version actually recorded in the manifest; anything
	64	// with alternatives, wildcards, or a non-registry source is skipped.
	65	func pin(spec string) string {
	66	s := strings.TrimSpace(spec)
	67	if s == "" \|\| strings.ContainsAny(s, "\|,* ") {
	68	return ""
	69	}
	70	for _, bad := range []string{"workspace:", "npm:", "file:", "link:", "git", "http", "://"} {
	71	if strings.Contains(s, bad) {
	72	return ""
	73	}
	74	}
	75	s = strings.TrimLeft(s, "^~>=<")
	76	s = strings.TrimPrefix(s, "v")
	77	if s == "" \|\| wildcard.MatchString(s) \|\| !exactVersion.MatchString(s) {
	78	return ""
	79	}
	80	return s
	81	}

internal/deps/manifest_test.go added +176

		@@ -0,0 +1,176 @@
	1	package deps
	2
	3	import (
	4	"fmt"
	5	"os"
	6	"testing"
	7	)
	8
	9	// tree serves manifests from a map, standing in for a git tree.
	10	func tree(files map[string]string) ReadFile {
	11	return func(path string) ([]byte, error) {
	12	body, ok := files[path]
	13	if !ok {
	14	return nil, os.ErrNotExist
	15	}
	16	return []byte(body), nil
	17	}
	18	}
	19
	20	func found(t *testing.T, deps []Dep) map[string]string {
	21	t.Helper()
	22	m := map[string]string{}
	23	for _, d := range deps {
	24	m[d.Ecosystem+":"+d.Name] = d.Current
	25	}
	26	return m
	27	}
	28
	29	func TestParseGoMod(t *testing.T) {
	30	deps := parseGoMod(tree(map[string]string{"go.mod": `
	31	module gitbay.org/gitbay
	32
	33	go 1.27.0
	34
	35	require (
	36	github.com/BurntSushi/toml v1.6.0
	37	golang.org/x/crypto v0.55.0
	38	github.com/vendored/thing v0.1.0
	39	)
	40
	41	require github.com/spf13/cobra v1.10.2
	42
	43	require (
	44	github.com/gorilla/css v1.0.1 // indirect
	45	golang.org/x/sys v0.47.0 // indirect
	46	)
	47
	48	replace github.com/vendored/thing => ./vendor/thing
	49	`}))
	50	got := found(t, deps)
	51	want := map[string]string{
	52	"go:github.com/BurntSushi/toml": "v1.6.0",
	53	"go:golang.org/x/crypto": "v0.55.0",
	54	"go:github.com/spf13/cobra": "v1.10.2",
	55	}
	56	if fmt.Sprint(got) != fmt.Sprint(want) {
	57	t.Errorf("go.mod deps = %v, want %v", got, want)
	58	}
	59	}
	60
	61	func TestParseNPMPrefersLockfile(t *testing.T) {
	62	files := map[string]string{
	63	"package.json": `{
	64	"dependencies": {"react": "^18.0.0", "left-pad": "1.3.0", "local": "file:../local"},
	65	"devDependencies": {"@types/node": "^20.0.0", "typescript": "*"}
	66	}`,
	67	"package-lock.json": `{
	68	"packages": {
	69	"": {"version": "1.0.0"},
	70	"node_modules/react": {"version": "18.2.0"},
	71	"node_modules/@types/node": {"version": "20.11.5"},
	72	"node_modules/react/node_modules/scheduler": {"version": "0.23.0"}
	73	}
	74	}`,
	75	}
	76	got := found(t, parseNPM(tree(files)))
	77	want := map[string]string{
	78	"npm:react": "18.2.0", // lockfile beats the ^18.0.0 floor
	79	"npm:@types/node": "20.11.5",
	80	"npm:left-pad": "1.3.0",
	81	}
	82	if fmt.Sprint(got) != fmt.Sprint(want) {
	83	t.Errorf("npm deps = %v, want %v", got, want)
	84	}
	85
	86	// Without a lockfile the declared floor is what there is to compare.
	87	delete(files, "package-lock.json")
	88	if got := found(t, parseNPM(tree(files)))["npm:react"]; got != "18.0.0" {
	89	t.Errorf("react without lockfile = %q, want 18.0.0", got)
	90	}
	91	}
	92
	93	func TestParseCargo(t *testing.T) {
	94	files := map[string]string{
	95	"Cargo.toml": `
	96	[dependencies]
	97	serde = "1.0.100"
	98	tokio = { version = "1.20", features = ["full"] }
	99	helper = { path = "../helper" }
	100	upstream = { git = "https://example.invalid/x" }
	101
	102	[dev-dependencies]
	103	criterion = "0.5"
	104	`,
	105	"Cargo.lock": `
	106	[[package]]
	107	name = "serde"
	108	version = "1.0.197"
	109
	110	[[package]]
	111	name = "tokio"
	112	version = "1.36.0"
	113	`,
	114	}
	115	got := found(t, parseCargo(tree(files)))
	116	want := map[string]string{
	117	"cargo:serde": "1.0.197",
	118	"cargo:tokio": "1.36.0",
	119	"cargo:criterion": "0.5", // not in the lock, so the requirement stands
	120	}
	121	if fmt.Sprint(got) != fmt.Sprint(want) {
	122	t.Errorf("cargo deps = %v, want %v", got, want)
	123	}
	124	}
	125
	126	func TestParsePython(t *testing.T) {
	127	files := map[string]string{
	128	"requirements.txt": `
	129	# comment
	130	requests==2.31.0
	131	django[bcrypt]==5.0.1 ; python_version >= "3.10"
	132	flask>=2,<3
	133	uvicorn
	134	-r other.txt
	135	`,
	136	"pyproject.toml": `
	137	[project]
	138	dependencies = ["httpx==0.27.0", "pydantic>=2"]
	139
	140	[tool.poetry.dependencies]
	141	python = "^3.11"
	142	rich = "^13.7.0"
	143	`,
	144	}
	145	got := found(t, parsePython(tree(files)))
	146	want := map[string]string{
	147	"pypi:requests": "2.31.0",
	148	"pypi:django": "5.0.1",
	149	"pypi:httpx": "0.27.0",
	150	"pypi:pydantic": "2",
	151	"pypi:rich": "13.7.0",
	152	}
	153	if fmt.Sprint(got) != fmt.Sprint(want) {
	154	t.Errorf("python deps = %v, want %v", got, want)
	155	}
	156	}
	157
	158	func TestScanSortsAndCaps(t *testing.T) {
	159	files := map[string]string{
	160	"go.mod": "module x\n\nrequire github.com/a/b v1.0.0\n",
	161	"package.json": `{"dependencies": {"z": "1.0.0"}}`,
	162	}
	163	got := Scan(tree(files))
	164	if len(got) != 2 {
	165	t.Fatalf("Scan = %v, want 2 deps", got)
	166	}
	167	if got[0].Ecosystem != EcoGo \|\| got[1].Ecosystem != EcoNPM {
	168	t.Errorf("Scan order = %s, %s", got[0].Ecosystem, got[1].Ecosystem)
	169	}
	170	}
	171
	172	func TestScanEmptyTree(t *testing.T) {
	173	if got := Scan(tree(nil)); len(got) != 0 {
	174	t.Errorf("Scan of empty tree = %v", got)
	175	}
	176	}

internal/deps/npm.go added +67

		@@ -0,0 +1,67 @@
	1	package deps
	2
	3	import (
	4	"encoding/json"
	5	"strings"
	6	)
	7
	8	// parseNPM takes the direct dependency names from package.json and their
	9	// installed versions from package-lock.json where one exists. The lockfile
	10	// is the honest source: package.json records ranges, and a range that
	11	// already admits the newest release is not something to report.
	12	func parseNPM(read ReadFile) []Dep {
	13	raw, err := read("package.json")
	14	if err != nil {
	15	return nil
	16	}
	17	var pkg struct {
	18	Dependencies map[string]string `json:"dependencies"`
	19	DevDependencies map[string]string `json:"devDependencies"`
	20	}
	21	if json.Unmarshal(raw, &pkg) != nil {
	22	return nil
	23	}
	24	locked := npmLock(read)
	25	var out []Dep
	26	for _, set := range []map[string]string{pkg.Dependencies, pkg.DevDependencies} {
	27	for name, spec := range set {
	28	current := locked[name]
	29	if current == "" {
	30	current = pin(spec)
	31	}
	32	if current != "" {
	33	out = append(out, Dep{Ecosystem: EcoNPM, Name: name, Current: current})
	34	}
	35	}
	36	}
	37	return out
	38	}
	39
	40	// npmLock reads installed versions out of a v2 or v3 lockfile, keyed by
	41	// package name. Nested entries (a transitive copy under another package's
	42	// node_modules) are ignored: only the top-level install is the direct one.
	43	func npmLock(read ReadFile) map[string]string {
	44	raw, err := read("package-lock.json")
	45	if err != nil {
	46	return nil
	47	}
	48	var lock struct {
	49	Packages map[string]struct {
	50	Version string `json:"version"`
	51	} `json:"packages"`
	52	}
	53	if json.Unmarshal(raw, &lock) != nil {
	54	return nil
	55	}
	56	out := map[string]string{}
	57	for path, entry := range lock.Packages {
	58	name, ok := strings.CutPrefix(path, "node_modules/")
	59	if !ok \|\| strings.Contains(name, "node_modules/") {
	60	continue
	61	}
	62	if v := pin(entry.Version); v != "" {
	63	out[name] = v
	64	}
	65	}
	66	return out
	67	}

internal/deps/python.go added +100

		@@ -0,0 +1,100 @@
	1	package deps
	2
	3	import (
	4	"strings"
	5
	6	"github.com/BurntSushi/toml"
	7	)
	8
	9	// parsePython reads requirements.txt and pyproject.toml. Only requirements
	10	// that name one release are reported: a bare `requests` or a `>=2,<3` range
	11	// says the project accepts whatever is current, so there is nothing to tell
	12	// its maintainer.
	13	func parsePython(read ReadFile) []Dep {
	14	seen := map[string]bool{}
	15	var out []Dep
	16	add := func(name, version string) {
	17	name = normalizePyPI(name)
	18	if name == "" \|\| name == "python" \|\| seen[name] {
	19	return
	20	}
	21	if v := pin(version); v != "" {
	22	seen[name] = true
	23	out = append(out, Dep{Ecosystem: EcoPyPI, Name: name, Current: v})
	24	}
	25	}
	26	if raw, err := read("requirements.txt"); err == nil {
	27	for _, line := range strings.Split(string(raw), "\n") {
	28	if i := strings.Index(line, "#"); i >= 0 {
	29	line = line[:i]
	30	}
	31	line = strings.TrimSpace(line)
	32	if line == "" \|\| strings.HasPrefix(line, "-") {
	33	continue
	34	}
	35	name, version := splitPEP508(line)
	36	add(name, version)
	37	}
	38	}
	39	if raw, err := read("pyproject.toml"); err == nil {
	40	var doc struct {
	41	Project struct {
	42	Dependencies []string `toml:"dependencies"`
	43	} `toml:"project"`
	44	Tool struct {
	45	Poetry struct {
	46	Dependencies map[string]any `toml:"dependencies"`
	47	DevDependencies map[string]any `toml:"dev-dependencies"`
	48	} `toml:"poetry"`
	49	} `toml:"tool"`
	50	}
	51	if toml.Unmarshal(raw, &doc) == nil {
	52	for _, spec := range doc.Project.Dependencies {
	53	name, version := splitPEP508(spec)
	54	add(name, version)
	55	}
	56	poetry := doc.Tool.Poetry
	57	for _, set := range []map[string]any{poetry.Dependencies, poetry.DevDependencies} {
	58	for name, spec := range set {
	59	// Poetry shares Cargo's two forms: a bare string, or a
	60	// table that may point somewhere other than PyPI.
	61	if req, ok := cargoRequirement(spec); ok {
	62	add(name, req)
	63	}
	64	}
	65	}
	66	}
	67	}
	68	return out
	69	}
	70
	71	// splitPEP508 separates the distribution name from its version specifier,
	72	// dropping extras and environment markers: `django[bcrypt]==5.0 ; sys_platform
	73	// != "win32"` becomes ("django", "==5.0").
	74	func splitPEP508(spec string) (name, version string) {
	75	if i := strings.Index(spec, ";"); i >= 0 {
	76	spec = spec[:i]
	77	}
	78	spec = strings.TrimSpace(spec)
	79	i := strings.IndexAny(spec, "[<>=!~ ")
	80	if i < 0 {
	81	return spec, ""
	82	}
	83	name, version = spec[:i], spec[i:]
	84	if j := strings.Index(version, "]"); j >= 0 {
	85	version = version[j+1:]
	86	}
	87	return name, strings.TrimSpace(version)
	88	}
	89
	90	// normalizePyPI applies PEP 503 name normalization, so `Flask_SQLAlchemy`
	91	// and `flask-sqlalchemy` are one dependency.
	92	func normalizePyPI(name string) string {
	93	name = strings.TrimSpace(strings.ToLower(name))
	94	name = strings.ReplaceAll(name, "_", "-")
	95	name = strings.ReplaceAll(name, ".", "-")
	96	for strings.Contains(name, "--") {
	97	name = strings.ReplaceAll(name, "--", "-")
	98	}
	99	return name
	100	}

internal/deps/registry.go added +137

		@@ -0,0 +1,137 @@
	1	package deps
	2
	3	import (
	4	"context"
	5	"encoding/json"
	6	"fmt"
	7	"io"
	8	"net/http"
	9	"regexp"
	10	"strings"
	11	"time"
	12	)
	13
	14	// Registry endpoints. Each is a fixed public host: unlike webhooks and
	15	// mirrors, no part of the URL comes from user input except the package
	16	// name, and safeName restricts that to characters that need no escaping in
	17	// a URL path.
	18	var endpoints = map[string]string{
	19	EcoGo: "https://proxy.golang.org",
	20	EcoNPM: "https://registry.npmjs.org",
	21	EcoCargo: "https://crates.io/api/v1/crates",
	22	EcoPyPI: "https://pypi.org/pypi",
	23	}
	24
	25	// maxBody bounds a registry response. The npm packument is the large one,
	26	// which is why the abbreviated metadata is requested.
	27	const maxBody = 4 << 20
	28
	29	// safeName is the shape a package name may have before it goes into a URL.
	30	var safeName = regexp.MustCompile(`^@?[A-Za-z0-9][A-Za-z0-9._/@+-]*$`)
	31
	32	// Client queries package registries. The zero value is not usable; call
	33	// NewClient.
	34	type Client struct {
	35	HTTP *http.Client
	36	Hosts map[string]string // overridden in tests
	37	Version string // reported in User-Agent
	38	}
	39
	40	func NewClient(version string) *Client {
	41	return &Client{
	42	HTTP: &http.Client{Timeout: 20 * time.Second},
	43	Hosts: endpoints,
	44	Version: version,
	45	}
	46	}
	47
	48	// Latest returns the current release of one package. A prerelease is
	49	// reported as no answer: nothing should be nudged onto an rc.
	50	func (c *Client) Latest(ctx context.Context, eco, name string) (string, error) {
	51	base, ok := c.Hosts[eco]
	52	if !ok {
	53	return "", fmt.Errorf("unknown ecosystem %q", eco)
	54	}
	55	if !safeName.MatchString(name) \|\| strings.Contains(name, "..") {
	56	return "", fmt.Errorf("unusable package name %q", name)
	57	}
	58	var path, accept string
	59	switch eco {
	60	case EcoGo:
	61	path = "/" + escapeModule(name) + "/@latest"
	62	case EcoNPM:
	63	path = "/" + name
	64	accept = "application/vnd.npm.install-v1+json" // dist-tags, not the full packument
	65	case EcoCargo:
	66	path = "/" + name
	67	case EcoPyPI:
	68	path = "/" + name + "/json"
	69	}
	70	body, err := c.get(ctx, base+path, accept)
	71	if err != nil {
	72	return "", err
	73	}
	74	var doc struct {
	75	Version string `json:"Version"` // go
	76	DistTags map[string]string `json:"dist-tags"` // npm
	77	Crate struct {
	78	MaxStableVersion string `json:"max_stable_version"`
	79	} `json:"crate"` // cargo
	80	Info struct {
	81	Version string `json:"version"`
	82	} `json:"info"` // pypi
	83	}
	84	if err := json.Unmarshal(body, &doc); err != nil {
	85	return "", fmt.Errorf("%s %s: %w", eco, name, err)
	86	}
	87	var latest string
	88	switch eco {
	89	case EcoGo:
	90	latest = doc.Version
	91	case EcoNPM:
	92	latest = doc.DistTags["latest"]
	93	case EcoCargo:
	94	latest = doc.Crate.MaxStableVersion
	95	case EcoPyPI:
	96	latest = doc.Info.Version
	97	}
	98	if latest == "" \|\| IsPrerelease(latest) {
	99	return "", nil
	100	}
	101	return latest, nil
	102	}
	103
	104	func (c *Client) get(ctx context.Context, u, accept string) ([]byte, error) {
	105	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
	106	if err != nil {
	107	return nil, err
	108	}
	109	req.Header.Set("User-Agent", "gitbay/"+c.Version+" (+https://gitbay.org)")
	110	if accept != "" {
	111	req.Header.Set("Accept", accept)
	112	}
	113	resp, err := c.HTTP.Do(req)
	114	if err != nil {
	115	return nil, err
	116	}
	117	defer resp.Body.Close()
	118	if resp.StatusCode != http.StatusOK {
	119	return nil, fmt.Errorf("%s: %s", u, resp.Status)
	120	}
	121	return io.ReadAll(io.LimitReader(resp.Body, maxBody))
	122	}
	123
	124	// escapeModule applies the module proxy's case encoding: an uppercase
	125	// letter becomes "!" followed by its lowercase form, so paths stay distinct
	126	// on case-insensitive filesystems.
	127	func escapeModule(path string) string {
	128	var b strings.Builder
	129	for _, r := range path {
	130	if r >= 'A' && r <= 'Z' {
	131	b.WriteByte('!')
	132	r += 'a' - 'A'
	133	}
	134	b.WriteRune(r)
	135	}
	136	return b.String()
	137	}

internal/deps/registry_test.go added +90

		@@ -0,0 +1,90 @@
	1	package deps
	2
	3	import (
	4	"context"
	5	"net/http"
	6	"net/http/httptest"
	7	"testing"
	8	)
	9
	10	// fakeRegistry serves one canned body per path and records what was asked
	11	// for, so the test can check the URL each ecosystem builds.
	12	func fakeRegistry(t testing.T, bodies map[string]string) (httptest.Server, *[]string) {
	13	t.Helper()
	14	var asked []string
	15	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	16	asked = append(asked, r.URL.Path)
	17	body, ok := bodies[r.URL.Path]
	18	if !ok {
	19	http.NotFound(w, r)
	20	return
	21	}
	22	w.Write([]byte(body))
	23	}))
	24	t.Cleanup(srv.Close)
	25	return srv, &asked
	26	}
	27
	28	func TestLatestPerEcosystem(t *testing.T) {
	29	srv, asked := fakeRegistry(t, map[string]string{
	30	"/github.com/!burnt!sushi/toml/@latest": `{"Version":"v1.6.0"}`,
	31	"/@types/node": `{"dist-tags":{"latest":"20.11.5"}}`,
	32	"/serde": `{"crate":{"max_stable_version":"1.0.197"}}`,
	33	"/requests/json": `{"info":{"version":"2.31.0"}}`,
	34	})
	35	c := NewClient("test")
	36	c.Hosts = map[string]string{EcoGo: srv.URL, EcoNPM: srv.URL, EcoCargo: srv.URL, EcoPyPI: srv.URL}
	37
	38	cases := []struct{ eco, name, want string }{
	39	{EcoGo, "github.com/BurntSushi/toml", "v1.6.0"},
	40	{EcoNPM, "@types/node", "20.11.5"},
	41	{EcoCargo, "serde", "1.0.197"},
	42	{EcoPyPI, "requests", "2.31.0"},
	43	}
	44	for _, c2 := range cases {
	45	got, err := c.Latest(context.Background(), c2.eco, c2.name)
	46	if err != nil {
	47	t.Errorf("Latest(%s, %s): %v", c2.eco, c2.name, err)
	48	continue
	49	}
	50	if got != c2.want {
	51	t.Errorf("Latest(%s, %s) = %q, want %q", c2.eco, c2.name, got, c2.want)
	52	}
	53	}
	54	if len(*asked) != 4 {
	55	t.Errorf("asked for %v", *asked)
	56	}
	57	}
	58
	59	func TestLatestSkipsPrerelease(t *testing.T) {
	60	srv, _ := fakeRegistry(t, map[string]string{"/x/json": `{"info":{"version":"2.0.0rc1"}}`})
	61	c := NewClient("test")
	62	c.Hosts = map[string]string{EcoPyPI: srv.URL}
	63	got, err := c.Latest(context.Background(), EcoPyPI, "x")
	64	if err != nil \|\| got != "" {
	65	t.Errorf("Latest = %q, %v; want no answer for a prerelease", got, err)
	66	}
	67	}
	68
	69	func TestLatestRejectsUnusableNames(t *testing.T) {
	70	srv, asked := fakeRegistry(t, nil)
	71	c := NewClient("test")
	72	c.Hosts = map[string]string{EcoGo: srv.URL}
	73	for _, name := range []string{"../../etc/passwd", "a/../b", "foo?bar", "foo bar", "", "-x"} {
	74	if _, err := c.Latest(context.Background(), EcoGo, name); err == nil {
	75	t.Errorf("Latest accepted %q", name)
	76	}
	77	}
	78	if len(*asked) != 0 {
	79	t.Errorf("unusable names reached the network: %v", *asked)
	80	}
	81	}
	82
	83	func TestLatestReportsHTTPError(t *testing.T) {
	84	srv, _ := fakeRegistry(t, nil)
	85	c := NewClient("test")
	86	c.Hosts = map[string]string{EcoCargo: srv.URL}
	87	if _, err := c.Latest(context.Background(), EcoCargo, "missing"); err == nil {
	88	t.Error("Latest on a 404 returned no error")
	89	}
	90	}

internal/deps/report.go added +63

		@@ -0,0 +1,63 @@
	1	package deps
	2
	3	import (
	4	"fmt"
	5	"sort"
	6	"strings"
	7
	8	"gitbay.org/gitbay/internal/store"
	9	)
	10
	11	// IssueTitle is fixed so the issue this worker maintains is recognizable
	12	// across sweeps, in listings, and to the maintainer.
	13	const IssueTitle = "Dependency updates available"
	14
	15	// ecosystemNames are how the four appear in the issue body.
	16	var ecosystemNames = map[string]string{
	17	EcoGo: "Go",
	18	EcoNPM: "npm",
	19	EcoCargo: "Cargo",
	20	EcoPyPI: "PyPI",
	21	}
	22
	23	// order is the sequence sections appear in.
	24	var order = []string{EcoGo, EcoNPM, EcoCargo, EcoPyPI}
	25
	26	// Body renders the issue: one table per ecosystem, sorted, so a diff
	27	// between sweeps reads as a diff of what is behind.
	28	func Body(branch string, reports []store.DepReport) string {
	29	var b strings.Builder
	30	fmt.Fprintf(&b, "Dependencies declared on `%s` are behind their latest release. "+
	31	"This issue is kept up to date as that changes, and closed once nothing is behind.\n", branch)
	32	byEco := map[string][]store.DepReport{}
	33	for _, r := range reports {
	34	byEco[r.Ecosystem] = append(byEco[r.Ecosystem], r)
	35	}
	36	for _, eco := range order {
	37	rows := byEco[eco]
	38	if len(rows) == 0 {
	39	continue
	40	}
	41	sort.Slice(rows, func(i, j int) bool { return rows[i].Name < rows[j].Name })
	42	fmt.Fprintf(&b, "\n### %s\n\n\| Package \| Current \| Latest \|\n\| --- \| --- \| --- \|\n", ecosystemNames[eco])
	43	for _, r := range rows {
	44	fmt.Fprintf(&b, "\| `%s` \| %s \| %s \|\n", r.Name, r.Current, r.Latest)
	45	}
	46	}
	47	return b.String()
	48	}
	49
	50	// same reports whether two outdated sets are equal, which is what decides
	51	// between leaving the issue alone and rewriting it. Both sides arrive
	52	// sorted by (ecosystem, name).
	53	func same(a, b []store.DepReport) bool {
	54	if len(a) != len(b) {
	55	return false
	56	}
	57	for i := range a {
	58	if a[i] != b[i] {
	59	return false
	60	}
	61	}
	62	return true
	63	}

internal/deps/version.go added +91

		@@ -0,0 +1,91 @@
	1	// Package deps reports dependencies that are behind their upstream
	2	// release. It reads manifests out of a repository's default branch and
	3	// asks the ecosystem's registry what the current version is; it never
	4	// executes a package manager, so the check needs no runner and no
	5	// per-repo configuration beyond opting in.
	6	package deps
	7
	8	import (
	9	"strconv"
	10	"strings"
	11	)
	12
	13	// Newer reports whether latest is a strictly greater release than current.
	14	// The four ecosystems agree on the part that matters here — dot-separated
	15	// numbers, optionally followed by a suffix — so one tolerant comparison
	16	// covers all of them. Anything it cannot read compares equal, which
	17	// reports nothing rather than reporting noise.
	18	func Newer(current, latest string) bool {
	19	ca, cs := split(current)
	20	la, ls := split(latest)
	21	if len(ca) == 0 \|\| len(la) == 0 {
	22	return false
	23	}
	24	for i := 0; i < len(ca) \|\| i < len(la); i++ {
	25	c, l := at(ca, i), at(la, i)
	26	if c != l {
	27	return l > c
	28	}
	29	}
	30	// Same release numbers: the suffix decides. A prerelease is behind the
	31	// plain release, which is behind a post-release.
	32	return rank(ls) > rank(cs)
	33	}
	34
	35	// IsPrerelease reports whether v carries a prerelease marker. Registries
	36	// mostly hand back stable versions already; this keeps the exceptions from
	37	// being suggested.
	38	func IsPrerelease(v string) bool {
	39	_, suffix := split(v)
	40	return rank(suffix) < 0
	41	}
	42
	43	// split separates the leading dot-separated numbers from whatever follows:
	44	// "v1.2.3-rc1" becomes ([1 2 3], "-rc1"), "2.0b1" becomes ([2 0], "b1").
	45	func split(v string) ([]int, string) {
	46	v = strings.TrimSpace(v)
	47	v = strings.TrimPrefix(v, "v")
	48	var nums []int
	49	i := 0
	50	for i < len(v) {
	51	j := i
	52	for j < len(v) && v[j] >= '0' && v[j] <= '9' {
	53	j++
	54	}
	55	if j == i {
	56	break
	57	}
	58	n, err := strconv.Atoi(v[i:j])
	59	if err != nil {
	60	break
	61	}
	62	nums = append(nums, n)
	63	if j < len(v) && v[j] == '.' && j+1 < len(v) && v[j+1] >= '0' && v[j+1] <= '9' {
	64	i = j + 1
	65	continue
	66	}
	67	i = j
	68	break
	69	}
	70	return nums, v[i:]
	71	}
	72
	73	func at(nums []int, i int) int {
	74	if i < len(nums) {
	75	return nums[i]
	76	}
	77	return 0
	78	}
	79
	80	// rank orders the three kinds of suffix a release can carry: -1 for a
	81	// prerelease, 0 for none, 1 for a PEP 440 post-release. Two prereleases
	82	// rank equal, which reports nothing rather than guessing at rc1 vs beta2.
	83	func rank(suffix string) int {
	84	switch {
	85	case suffix == "":
	86	return 0
	87	case strings.HasPrefix(strings.TrimLeft(suffix, ".-_"), "post"):
	88	return 1
	89	}
	90	return -1
	91	}

internal/deps/version_test.go added +72

		@@ -0,0 +1,72 @@
	1	package deps
	2
	3	import "testing"
	4
	5	func TestNewer(t *testing.T) {
	6	cases := []struct {
	7	current, latest string
	8	want bool
	9	}{
	10	{"v1.2.3", "v1.3.0", true},
	11	{"v1.2.3", "v1.2.3", false},
	12	{"v1.3.0", "v1.2.3", false},
	13	{"1.2", "1.2.1", true},
	14	{"1.2.1", "1.2", false},
	15	{"0.9", "1.0", true},
	16	{"1.9.0", "1.10.0", true}, // numeric, not lexical
	17	{"2.0.0-rc1", "2.0.0", true},
	18	{"2.0.0", "2.0.0-rc1", false},
	19	{"1.2.3", "1.2.3.post1", true},
	20	{"1.2.3.post1", "1.2.3", false},
	21	{"2.0b1", "2.0", true},
	22	// A pseudo-version is behind any tagged release.
	23	{"v0.0.0-20230129092748-24d4a6f8daec", "v1.0.0", true},
	24	// Unreadable input reports nothing rather than guessing.
	25	{"", "1.0.0", false},
	26	{"1.0.0", "", false},
	27	{"main", "1.0.0", false},
	28	}
	29	for _, c := range cases {
	30	if got := Newer(c.current, c.latest); got != c.want {
	31	t.Errorf("Newer(%q, %q) = %v, want %v", c.current, c.latest, got, c.want)
	32	}
	33	}
	34	}
	35
	36	func TestIsPrerelease(t *testing.T) {
	37	for _, v := range []string{"2.0.0-rc1", "1.0.0-beta.2", "2.0b1", "1.0.0-alpha"} {
	38	if !IsPrerelease(v) {
	39	t.Errorf("IsPrerelease(%q) = false", v)
	40	}
	41	}
	42	for _, v := range []string{"1.2.3", "v1.2.3", "1.2.3.post1", "10.0"} {
	43	if IsPrerelease(v) {
	44	t.Errorf("IsPrerelease(%q) = true", v)
	45	}
	46	}
	47	}
	48
	49	func TestPin(t *testing.T) {
	50	cases := map[string]string{
	51	"1.2.3": "1.2.3",
	52	"^1.2.3": "1.2.3",
	53	"~1.2": "1.2",
	54	">=2.0.0": "2.0.0",
	55	"==1.4.2": "1.4.2",
	56	"v1.2.3": "1.2.3",
	57	"1.2.3-rc1": "1.2.3-rc1",
	58	"*": "",
	59	">=2,<3": "",
	60	"1.x": "",
	61	"^1.0 \|\| ^2.0": "",
	62	"workspace:*": "",
	63	"npm:foo@1.0": "",
	64	"file:../lib": "",
	65	"": "",
	66	}
	67	for spec, want := range cases {
	68	if got := pin(spec); got != want {
	69	t.Errorf("pin(%q) = %q, want %q", spec, got, want)
	70	}
	71	}
	72	}

internal/deps/worker.go added +245

		@@ -0,0 +1,245 @@
	1	package deps
	2
	3	import (
	4	"context"
	5	"errors"
	6	"fmt"
	7	"log/slog"
	8	"os"
	9	"sort"
	10	"strings"
	11	"sync"
	12	"time"
	13
	14	"gitbay.org/gitbay/internal/config"
	15	"gitbay.org/gitbay/internal/gitutil"
	16	"gitbay.org/gitbay/internal/store"
	17	)
	18
	19	// manifestLimit bounds one manifest read out of the object store.
	20	const manifestLimit = 1 << 20
	21
	22	// lookups is how many registry requests one repository has in flight.
	23	const lookups = 4
	24
	25	// Worker sweeps repositories that have opted in, comparing their manifests
	26	// against the registries and maintaining one issue per repository.
	27	type Worker struct {
	28	St *store.Store
	29	Cfg config.Config
	30	RepoDir func(owner, name string) string
	31	Client *Client
	32	Tick time.Duration
	33	}
	34
	35	func New(st store.Store, cfg config.Config, repoDir func(owner, name string) string, version string) Worker {
	36	tick := time.Hour
	37	if v := os.Getenv("GITBAY_DEPS_TICK"); v != "" {
	38	if d, err := time.ParseDuration(v); err == nil {
	39	tick = d
	40	}
	41	}
	42	return &Worker{St: st, Cfg: cfg, RepoDir: repoDir, Client: NewClient(version), Tick: tick}
	43	}
	44
	45	// Run sweeps until ctx ends.
	46	func (w *Worker) Run(ctx context.Context) {
	47	t := time.NewTicker(w.Tick)
	48	defer t.Stop()
	49	for {
	50	select {
	51	case <-ctx.Done():
	52	return
	53	case <-t.C:
	54	w.Sweep(ctx)
	55	}
	56	}
	57	}
	58
	59	// Sweep checks every repository whose interval has elapsed. Split from the
	60	// ticker for tests.
	61	func (w *Worker) Sweep(ctx context.Context) {
	62	interval := w.Cfg.Deps.CheckIntervalHours
	63	if interval <= 0 {
	64	interval = 24
	65	}
	66	due, err := w.St.DueDepChecks(interval * 3600)
	67	if err != nil {
	68	slog.Error("deps: listing due repos", "err", err)
	69	return
	70	}
	71	for _, repo := range due {
	72	if ctx.Err() != nil {
	73	return
	74	}
	75	msg := ""
	76	if err := w.check(ctx, repo); err != nil {
	77	slog.Warn("deps check failed", "repo", repo.Path(), "err", err)
	78	msg = err.Error()
	79	}
	80	// Stamped either way, so a repo that fails every time is retried on
	81	// the interval rather than on every sweep.
	82	w.St.SetDepCheckResult(repo.ID, msg)
	83	}
	84	}
	85
	86	// check compares one repository against the registries and reconciles its
	87	// issue with the result.
	88	func (w *Worker) check(ctx context.Context, repo store.Repo) error {
	89	dir := w.RepoDir(repo.OwnerName, repo.Name)
	90	sha, err := gitutil.ResolveRef(dir, "refs/heads/"+repo.DefaultBranch)
	91	if err != nil {
	92	return nil // empty repo, or no default branch yet
	93	}
	94	found := Scan(func(path string) ([]byte, error) {
	95	return gitutil.ReadBlob(dir, sha, path, manifestLimit)
	96	})
	97	if len(found) == 0 {
	98	return w.reconcile(repo, nil)
	99	}
	100	behind, err := w.behind(ctx, found)
	101	if err != nil {
	102	return err
	103	}
	104	return w.reconcile(repo, behind)
	105	}
	106
	107	// behind queries the registries and keeps the dependencies whose latest
	108	// release is greater than what the manifest declares. A lookup that fails
	109	// is dropped rather than failing the sweep: one unreachable package should
	110	// not silence the rest, and the next sweep tries again.
	111	func (w *Worker) behind(ctx context.Context, found []Dep) ([]store.DepReport, error) {
	112	var (
	113	mu sync.Mutex
	114	out []store.DepReport
	115	errs []string
	116	wg sync.WaitGroup
	117	tickets = make(chan struct{}, lookups)
	118	)
	119	for _, d := range found {
	120	wg.Add(1)
	121	go func(d Dep) {
	122	defer wg.Done()
	123	tickets <- struct{}{}
	124	defer func() { <-tickets }()
	125	latest, err := w.Client.Latest(ctx, d.Ecosystem, d.Name)
	126	mu.Lock()
	127	defer mu.Unlock()
	128	switch {
	129	case err != nil:
	130	errs = append(errs, fmt.Sprintf("%s %s: %v", d.Ecosystem, d.Name, err))
	131	case latest != "" && Newer(d.Current, latest):
	132	out = append(out, store.DepReport{
	133	Ecosystem: d.Ecosystem, Name: d.Name, Current: d.Current, Latest: latest})
	134	}
	135	}(d)
	136	}
	137	wg.Wait()
	138	if ctx.Err() != nil {
	139	return nil, ctx.Err()
	140	}
	141	// Every lookup failing means the registries are unreachable, which is
	142	// worth recording; a few failing is ordinary.
	143	if len(errs) == len(found) && len(errs) > 0 {
	144	return nil, errors.New(errs[0])
	145	}
	146	sort.Slice(out, func(i, j int) bool {
	147	if out[i].Ecosystem != out[j].Ecosystem {
	148	return out[i].Ecosystem < out[j].Ecosystem
	149	}
	150	return out[i].Name < out[j].Name
	151	})
	152	return out, nil
	153	}
	154
	155	// reconcile brings the repository's issue in line with what is behind:
	156	// opened when something first falls behind, rewritten when the set changes,
	157	// closed when nothing is behind any more.
	158	func (w *Worker) reconcile(repo store.Repo, behind []store.DepReport) error {
	159	check, err := w.St.DepCheckFor(repo.ID)
	160	if err != nil {
	161	return err
	162	}
	163	previous, err := w.St.ReportedDeps(repo.ID)
	164	if err != nil {
	165	return err
	166	}
	167	issue, hasIssue := w.openIssue(repo, check.IssueNumber)
	168	if len(behind) == 0 {
	169	if hasIssue {
	170	w.St.SetIssueState(issue.ID, "closed")
	171	w.St.SetDepIssue(repo.ID, 0)
	172	}
	173	if len(previous) > 0 {
	174	return w.St.ReplaceDepReports(repo.ID, nil)
	175	}
	176	return nil
	177	}
	178	// Nothing changed since the last report: leave it be, whether the issue
	179	// is still open or the maintainer has closed it. Reopening on an
	180	// unchanged set would make closing the issue pointless.
	181	if same(previous, behind) {
	182	return nil
	183	}
	184	if err := w.St.ReplaceDepReports(repo.ID, behind); err != nil {
	185	return err
	186	}
	187	body := Body(repo.DefaultBranch, behind)
	188	if hasIssue {
	189	if err := w.St.UpdateIssueText(issue.ID, nil, &body, nil); err != nil {
	190	return err
	191	}
	192	w.notify(repo, issue.Number, fmt.Sprintf("updated issue #%d", issue.Number), body)
	193	return nil
	194	}
	195	author, err := w.St.UserByUsername(store.BotUsername)
	196	if err != nil {
	197	return fmt.Errorf("loading %s: %w", store.BotUsername, err)
	198	}
	199	number, err := w.St.CreateIssue(repo.ID, author.ID, IssueTitle, body, "md")
	200	if err != nil {
	201	return err
	202	}
	203	if err := w.St.SetDepIssue(repo.ID, number); err != nil {
	204	return err
	205	}
	206	w.St.RecordEvent(repo.ID, author.ID, "issue.created", fmt.Sprintf(`{"number":%d}`, number))
	207	w.notify(repo, number, fmt.Sprintf("opened issue #%d", number), body)
	208	return nil
	209	}
	210
	211	// openIssue loads the issue this worker maintains, if there still is one.
	212	// A closed issue counts as gone: reopening one the maintainer closed would
	213	// be arguing with them, so the next change opens a fresh issue.
	214	func (w *Worker) openIssue(repo store.Repo, number int64) (store.Issue, bool) {
	215	if number == 0 {
	216	return store.Issue{}, false
	217	}
	218	issue, err := w.St.IssueByNumber(repo.ID, number)
	219	if err != nil \|\| issue.State != "open" {
	220	return store.Issue{}, false
	221	}
	222	return issue, true
	223	}
	224
	225	// notify mails the repo's owners, the same targets and shape as an issue
	226	// filed over SSH. Best-effort, like every other notification.
	227	func (w *Worker) notify(repo store.Repo, number int64, action, body string) {
	228	if w.Cfg.Mail.SMTPHost == "" {
	229	return
	230	}
	231	targets, err := w.St.RepoNotifyTargets(repo)
	232	if err != nil {
	233	return
	234	}
	235	subject := fmt.Sprintf("[%s] #%d: %s", repo.Path(), number, IssueTitle)
	236	text := fmt.Sprintf("%s %s\n\n%s\n%s/%s/issues/%d\n", store.BotUsername, action, body,
	237	strings.TrimSuffix(w.Cfg.Server.SiteURL, "/"), repo.Path(), number)
	238	for _, id := range targets {
	239	email, err := w.St.PrimaryVerifiedEmail(id)
	240	if err != nil \|\| email == "" {
	241	continue
	242	}
	243	w.St.EnqueueMail(email, subject, text)
	244	}
	245	}

internal/deps/worker_test.go added +225

		@@ -0,0 +1,225 @@
	1	package deps
	2
	3	import (
	4	"context"
	5	"path/filepath"
	6	"strings"
	7	"testing"
	8
	9	"gitbay.org/gitbay/internal/config"
	10	"gitbay.org/gitbay/internal/store"
	11	)
	12
	13	func testWorker(t testing.T) (Worker, store.Repo) {
	14	t.Helper()
	15	st, err := store.Open(filepath.Join(t.TempDir(), "gitbay.db"))
	16	if err != nil {
	17	t.Fatal(err)
	18	}
	19	t.Cleanup(func() { st.Close() })
	20	if err := st.MigrateUp(); err != nil {
	21	t.Fatal(err)
	22	}
	23	owner, err := st.CreateUser("alice", false)
	24	if err != nil {
	25	t.Fatal(err)
	26	}
	27	if err := st.AddEmail(owner, "alice@example.com", "admin", true); err != nil {
	28	t.Fatal(err)
	29	}
	30	id, err := st.CreateRepo("user", owner, "thing", "public")
	31	if err != nil {
	32	t.Fatal(err)
	33	}
	34	repo, err := st.RepoByID(id)
	35	if err != nil {
	36	t.Fatal(err)
	37	}
	38	if err := st.EnableDepCheck(repo.ID); err != nil {
	39	t.Fatal(err)
	40	}
	41	cfg := config.Default()
	42	cfg.Server.SiteURL = "https://gitbay.test"
	43	cfg.Mail.SMTPHost = "localhost:587"
	44	return &Worker{St: st, Cfg: cfg}, repo
	45	}
	46
	47	func reports(pairs ...string) []store.DepReport {
	48	var out []store.DepReport
	49	for i := 0; i < len(pairs); i += 3 {
	50	out = append(out, store.DepReport{
	51	Ecosystem: EcoGo, Name: pairs[i], Current: pairs[i+1], Latest: pairs[i+2]})
	52	}
	53	return out
	54	}
	55
	56	func TestReconcileIssueLifecycle(t *testing.T) {
	57	w, repo := testWorker(t)
	58
	59	// Nothing behind: no issue, no mail.
	60	if err := w.reconcile(repo, nil); err != nil {
	61	t.Fatal(err)
	62	}
	63	if check, _ := w.St.DepCheckFor(repo.ID); check.IssueNumber != 0 {
	64	t.Fatalf("issue %d opened with nothing behind", check.IssueNumber)
	65	}
	66
	67	// Something falls behind: one issue, one notification.
	68	if err := w.reconcile(repo, reports("github.com/a/b", "v1.0.0", "v1.1.0")); err != nil {
	69	t.Fatal(err)
	70	}
	71	check, err := w.St.DepCheckFor(repo.ID)
	72	if err != nil \|\| check.IssueNumber == 0 {
	73	t.Fatalf("no issue opened: %v", err)
	74	}
	75	issue, err := w.St.IssueByNumber(repo.ID, check.IssueNumber)
	76	if err != nil {
	77	t.Fatal(err)
	78	}
	79	if issue.Author != store.BotUsername {
	80	t.Errorf("issue author = %q, want %q", issue.Author, store.BotUsername)
	81	}
	82	if issue.Title != IssueTitle {
	83	t.Errorf("issue title = %q", issue.Title)
	84	}
	85	if !strings.Contains(issue.Body, "github.com/a/b") \|\| !strings.Contains(issue.Body, "v1.1.0") {
	86	t.Errorf("issue body missing the dependency:\n%s", issue.Body)
	87	}
	88	mail, err := w.St.DueMail(10)
	89	if err != nil {
	90	t.Fatal(err)
	91	}
	92	if len(mail) != 1 \|\| mail[0].Recipient != "alice@example.com" {
	93	t.Fatalf("mail = %+v, want one to the owner", mail)
	94	}
	95
	96	// Same set again: the issue is left alone and nobody is mailed twice.
	97	if err := w.reconcile(repo, reports("github.com/a/b", "v1.0.0", "v1.1.0")); err != nil {
	98	t.Fatal(err)
	99	}
	100	if mail, _ := w.St.DueMail(10); len(mail) != 1 {
	101	t.Errorf("unchanged set mailed again: %d messages", len(mail))
	102	}
	103
	104	// The set changes: same issue, rewritten body, another notification.
	105	if err := w.reconcile(repo, reports("github.com/a/b", "v1.0.0", "v1.2.0")); err != nil {
	106	t.Fatal(err)
	107	}
	108	after, _ := w.St.DepCheckFor(repo.ID)
	109	if after.IssueNumber != check.IssueNumber {
	110	t.Errorf("second issue opened: %d then %d", check.IssueNumber, after.IssueNumber)
	111	}
	112	issue, _ = w.St.IssueByNumber(repo.ID, check.IssueNumber)
	113	if !strings.Contains(issue.Body, "v1.2.0") {
	114	t.Errorf("issue body not rewritten:\n%s", issue.Body)
	115	}
	116	if mail, _ := w.St.DueMail(10); len(mail) != 2 {
	117	t.Errorf("changed set produced %d messages, want 2", len(mail))
	118	}
	119
	120	// Caught up: the issue closes and the reports are forgotten.
	121	if err := w.reconcile(repo, nil); err != nil {
	122	t.Fatal(err)
	123	}
	124	issue, _ = w.St.IssueByNumber(repo.ID, check.IssueNumber)
	125	if issue.State != "closed" {
	126	t.Errorf("issue state = %q, want closed", issue.State)
	127	}
	128	if left, _ := w.St.ReportedDeps(repo.ID); len(left) != 0 {
	129	t.Errorf("reports left behind: %v", left)
	130	}
	131	if mail, _ := w.St.DueMail(10); len(mail) != 2 {
	132	t.Errorf("closing mailed: %d messages", len(mail))
	133	}
	134	}
	135
	136	func TestReconcileOpensFreshIssueAfterClose(t *testing.T) {
	137	w, repo := testWorker(t)
	138	if err := w.reconcile(repo, reports("github.com/a/b", "v1.0.0", "v1.1.0")); err != nil {
	139	t.Fatal(err)
	140	}
	141	first, _ := w.St.DepCheckFor(repo.ID)
	142	issue, _ := w.St.IssueByNumber(repo.ID, first.IssueNumber)
	143
	144	// The maintainer closes it. The worker does not reopen it; the next
	145	// change gets its own issue.
	146	if err := w.St.SetIssueState(issue.ID, "closed"); err != nil {
	147	t.Fatal(err)
	148	}
	149	if err := w.reconcile(repo, reports("github.com/a/b", "v1.0.0", "v1.3.0")); err != nil {
	150	t.Fatal(err)
	151	}
	152	second, _ := w.St.DepCheckFor(repo.ID)
	153	if second.IssueNumber == first.IssueNumber {
	154	t.Fatalf("reused closed issue #%d", first.IssueNumber)
	155	}
	156	if reopened, _ := w.St.IssueByNumber(repo.ID, first.IssueNumber); reopened.State != "closed" {
	157	t.Error("the closed issue was reopened")
	158	}
	159	}
	160
	161	func TestBehindQueriesRegistries(t *testing.T) {
	162	srv, _ := fakeRegistry(t, map[string]string{
	163	"/github.com/a/b/@latest": `{"Version":"v1.1.0"}`,
	164	"/github.com/c/d/@latest": `{"Version":"v2.0.0"}`,
	165	})
	166	w, _ := testWorker(t)
	167	w.Client = NewClient("test")
	168	w.Client.Hosts = map[string]string{EcoGo: srv.URL}
	169
	170	got, err := w.behind(context.Background(), []Dep{
	171	{Ecosystem: EcoGo, Name: "github.com/a/b", Current: "v1.0.0"}, // behind
	172	{Ecosystem: EcoGo, Name: "github.com/c/d", Current: "v2.0.0"}, // current
	173	})
	174	if err != nil {
	175	t.Fatal(err)
	176	}
	177	if len(got) != 1 \|\| got[0].Name != "github.com/a/b" \|\| got[0].Latest != "v1.1.0" {
	178	t.Fatalf("behind = %+v", got)
	179	}
	180	}
	181
	182	func TestBehindToleratesOneFailureButNotAll(t *testing.T) {
	183	srv, _ := fakeRegistry(t, map[string]string{"/github.com/a/b/@latest": `{"Version":"v1.1.0"}`})
	184	w, _ := testWorker(t)
	185	w.Client = NewClient("test")
	186	w.Client.Hosts = map[string]string{EcoGo: srv.URL}
	187	found := []Dep{
	188	{Ecosystem: EcoGo, Name: "github.com/a/b", Current: "v1.0.0"},
	189	{Ecosystem: EcoGo, Name: "github.com/gone/away", Current: "v1.0.0"}, // 404s
	190	}
	191	got, err := w.behind(context.Background(), found)
	192	if err != nil {
	193	t.Fatalf("one failed lookup failed the sweep: %v", err)
	194	}
	195	if len(got) != 1 {
	196	t.Fatalf("behind = %+v", got)
	197	}
	198	if _, err := w.behind(context.Background(), found[1:]); err == nil {
	199	t.Error("every lookup failing was reported as success")
	200	}
	201	}
	202
	203	func TestReconcileLeavesClosedIssueClosedOnUnchangedSet(t *testing.T) {
	204	w, repo := testWorker(t)
	205	behind := reports("github.com/a/b", "v1.0.0", "v1.1.0")
	206	if err := w.reconcile(repo, behind); err != nil {
	207	t.Fatal(err)
	208	}
	209	check, _ := w.St.DepCheckFor(repo.ID)
	210	issue, _ := w.St.IssueByNumber(repo.ID, check.IssueNumber)
	211	if err := w.St.SetIssueState(issue.ID, "closed"); err != nil {
	212	t.Fatal(err)
	213	}
	214	// Nothing has changed, so closing the issue has to stick.
	215	if err := w.reconcile(repo, behind); err != nil {
	216	t.Fatal(err)
	217	}
	218	after, _ := w.St.DepCheckFor(repo.ID)
	219	if after.IssueNumber != check.IssueNumber {
	220	t.Fatalf("opened issue #%d on an unchanged set", after.IssueNumber)
	221	}
	222	if again, _ := w.St.IssueByNumber(repo.ID, check.IssueNumber); again.State != "closed" {
	223	t.Error("the closed issue came back")
	224	}
	225	}

internal/httpd/settings.go +13 −4

		@@ -17,9 +17,10 @@ import (
17	17
18	18	type settingsPage struct {
19	19	repoPage
20		Topics []string
21		Branches []gitutil.Ref
22		Notice string
	20	Topics []string
	21	Branches []gitutil.Ref
	22	DepsEnabled bool
	23	Notice string
23	24	}
24	25
25	26	func (s Server) settingsForm(w http.ResponseWriter, r http.Request, u store.User) {
		@@ -34,9 +35,11 @@ func (s Server) settingsForm(w http.ResponseWriter, r http.Request, u store.Us
34	35	p.Tab = "settings"
35	36	topics, _ := s.st.ListTopics(repo.ID)
36	37	branches, _ := gitutil.Refs(p.Dir, "heads")
	38	_, depsErr := s.st.DepCheckFor(repo.ID)
37	39	s.render(w, "settings.html", settingsPage{
38	40	repoPage: p, Topics: topics, Branches: branches,
39		Notice: r.URL.Query().Get("e"),
	41	DepsEnabled: depsErr == nil,
	42	Notice: r.URL.Query().Get("e"),
40	43	})
41	44	}
42	45
		@@ -79,6 +82,12 @@ func (s Server) settingsSubmit(w http.ResponseWriter, r http.Request, u store.
79	82	argv = []string{"repo", "settings", "protect", repo, v("branch")}
80	83	case "unprotect":
81	84	argv = []string{"repo", "settings", "unprotect", repo, v("branch")}
	85	case "deps":
	86	verb := "disable"
	87	if v("deps") == "on" {
	88	verb = "enable"
	89	}
	90	argv = []string{"repo", "deps", verb, repo}
82	91	case "archive":
83	92	verb := "archive"
84	93	if v("archive") != "on" {

internal/policy/names.go +1

		@@ -17,6 +17,7 @@ var reservedNames = map[string]bool{
17	17	"explore": true,
18	18	"favicon.svg": true,
19	19	"gitbay": true, // vanity go-import path on gitbay.org
	20	"gitbay-bot": true, // authors dependency-update issues
20	21	"login": true,
21	22	"logout": true,
22	23	"new": true,

internal/store/deps.go added +142

		@@ -0,0 +1,142 @@
	1	package store
	2
	3	import (
	4	"database/sql"
	5	"errors"
	6	)
	7
	8	// BotUsername authors dependency-update issues. The account exists from
	9	// migration 0028 with no key and no email: it authors, it never
	10	// authenticates.
	11	const BotUsername = "gitbay-bot"
	12
	13	// DepCheck is a repo's opt-in dependency sweep state. A row exists only
	14	// while checking is enabled.
	15	type DepCheck struct {
	16	RepoID int64
	17	LastCheck string
	18	LastError string
	19	IssueNumber int64 // 0 until an issue has been opened
	20	}
	21
	22	// DepReport is one dependency found to be behind, as last reported.
	23	type DepReport struct {
	24	Ecosystem string
	25	Name string
	26	Current string
	27	Latest string
	28	}
	29
	30	func (s *Store) EnableDepCheck(repoID int64) error {
	31	_, err := s.DB.Exec("INSERT OR IGNORE INTO dep_checks (repo_id) VALUES (?)", repoID)
	32	return err
	33	}
	34
	35	// DisableDepCheck stops checking and forgets what was reported, so
	36	// re-enabling reports the current state afresh.
	37	func (s *Store) DisableDepCheck(repoID int64) error {
	38	tx, err := s.DB.Begin()
	39	if err != nil {
	40	return err
	41	}
	42	defer tx.Rollback()
	43	if _, err := tx.Exec("DELETE FROM dep_reports WHERE repo_id = ?", repoID); err != nil {
	44	return err
	45	}
	46	if _, err := tx.Exec("DELETE FROM dep_checks WHERE repo_id = ?", repoID); err != nil {
	47	return err
	48	}
	49	return tx.Commit()
	50	}
	51
	52	func (s *Store) DepCheckFor(repoID int64) (DepCheck, error) {
	53	var d DepCheck
	54	err := s.DB.QueryRow(
	55	"SELECT repo_id, last_check, last_error, issue_number FROM dep_checks WHERE repo_id = ?", repoID).
	56	Scan(&d.RepoID, &d.LastCheck, &d.LastError, &d.IssueNumber)
	57	if errors.Is(err, sql.ErrNoRows) {
	58	return d, ErrNotFound
	59	}
	60	return d, err
	61	}
	62
	63	// DueDepChecks returns repos whose last check is older than
	64	// intervalSeconds. Archived repos are skipped: nobody is going to act on
	65	// the issue.
	66	func (s *Store) DueDepChecks(intervalSeconds int) ([]Repo, error) {
	67	rows, err := s.DB.Query(repoSelect+`
	68	JOIN dep_checks d ON d.repo_id = r.id
	69	WHERE d.last_check = ''
	70	OR strftime('%s','now') - strftime('%s', d.last_check) > ?
	71	ORDER BY r.id`, intervalSeconds)
	72	if err != nil {
	73	return nil, err
	74	}
	75	defer rows.Close()
	76	var out []Repo
	77	for rows.Next() {
	78	r, err := scanRepo(rows)
	79	if err != nil {
	80	return nil, err
	81	}
	82	if r.Settings.Archived {
	83	continue
	84	}
	85	out = append(out, r)
	86	}
	87	return out, rows.Err()
	88	}
	89
	90	// SetDepCheckResult stamps a sweep. An empty checkErr records success.
	91	func (s *Store) SetDepCheckResult(repoID int64, checkErr string) error {
	92	_, err := s.DB.Exec(`
	93	UPDATE dep_checks SET last_error = ?,
	94	last_check = strftime('%Y-%m-%dT%H:%M:%fZ','now')
	95	WHERE repo_id = ?`, checkErr, repoID)
	96	return err
	97	}
	98
	99	func (s *Store) SetDepIssue(repoID, number int64) error {
	100	_, err := s.DB.Exec("UPDATE dep_checks SET issue_number = ? WHERE repo_id = ?", number, repoID)
	101	return err
	102	}
	103
	104	func (s *Store) ReportedDeps(repoID int64) ([]DepReport, error) {
	105	rows, err := s.DB.Query(`
	106	SELECT ecosystem, name, current, latest FROM dep_reports
	107	WHERE repo_id = ? ORDER BY ecosystem, name`, repoID)
	108	if err != nil {
	109	return nil, err
	110	}
	111	defer rows.Close()
	112	var out []DepReport
	113	for rows.Next() {
	114	var d DepReport
	115	if err := rows.Scan(&d.Ecosystem, &d.Name, &d.Current, &d.Latest); err != nil {
	116	return nil, err
	117	}
	118	out = append(out, d)
	119	}
	120	return out, rows.Err()
	121	}
	122
	123	// ReplaceDepReports swaps in the current outdated set wholesale: a
	124	// dependency that was updated, removed, or renamed leaves no trace.
	125	func (s *Store) ReplaceDepReports(repoID int64, reports []DepReport) error {
	126	tx, err := s.DB.Begin()
	127	if err != nil {
	128	return err
	129	}
	130	defer tx.Rollback()
	131	if _, err := tx.Exec("DELETE FROM dep_reports WHERE repo_id = ?", repoID); err != nil {
	132	return err
	133	}
	134	for _, d := range reports {
	135	if _, err := tx.Exec(`
	136	INSERT INTO dep_reports (repo_id, ecosystem, name, current, latest)
	137	VALUES (?, ?, ?, ?, ?)`, repoID, d.Ecosystem, d.Name, d.Current, d.Latest); err != nil {
	138	return err
	139	}
	140	}
	141	return tx.Commit()
	142	}

internal/store/migrations/0028_deps.down.sql added +3

		@@ -0,0 +1,3 @@
	1	DELETE FROM users WHERE username = 'gitbay-bot';
	2	DROP TABLE dep_reports;
	3	DROP TABLE dep_checks;

internal/store/migrations/0028_deps.up.sql added +25

		@@ -0,0 +1,25 @@
	1	-- Dependency update checks: an opt-in per-repo sweep that compares the
	2	-- manifests on the default branch against upstream registries and reports
	3	-- what is behind in an issue. Opt-in because checking a private repo tells
	4	-- a public registry what it depends on.
	5	CREATE TABLE dep_checks (
	6	repo_id INTEGER PRIMARY KEY REFERENCES repos(id) ON DELETE CASCADE,
	7	last_check TEXT NOT NULL DEFAULT '',
	8	last_error TEXT NOT NULL DEFAULT '',
	9	issue_number INTEGER NOT NULL DEFAULT 0
	10	);
	11
	12	-- What each repo was last told about, so a repo that stays behind is
	13	-- reported once rather than every sweep.
	14	CREATE TABLE dep_reports (
	15	repo_id INTEGER NOT NULL REFERENCES repos(id) ON DELETE CASCADE,
	16	ecosystem TEXT NOT NULL,
	17	name TEXT NOT NULL,
	18	current TEXT NOT NULL,
	19	latest TEXT NOT NULL,
	20	PRIMARY KEY (repo_id, ecosystem, name)
	21	);
	22
	23	-- The account dependency issues are authored by. Keyless and mailless: it
	24	-- authors, it never authenticates.
	25	INSERT INTO users (username, is_admin) VALUES ('gitbay-bot', 0);

internal/web/templates/settings.html +12

		@@ -91,6 +91,18 @@
91	91	<button type="submit">Protect</button>
92	92	</form>
93	93
	94	<h2>Dependencies</h2>
	95	<form method="post" action="{{$base}}" class="setform">
	96	<input type="hidden" name="field" value="deps">
	97	<label for="deps">Check for updates</label>
	98	<input type="checkbox" id="deps" name="deps" value="on"{{if .DepsEnabled}} checked{{end}}>
	99	<button type="submit">Save</button>
	100	</form>
	101	<p class="meta">Compares the manifests on <code>{{.Repo.DefaultBranch}}</code> against
	102	proxy.golang.org, npm, crates.io, and PyPI once a day, and tracks what is behind
	103	in an issue. Checking a private repository tells those registries what it
	104	depends on.</p>
	105
94	106	<h2>Lifecycle</h2>
95	107	<form method="post" action="{{$base}}" class="setform">
96	108	<input type="hidden" name="field" value="archive">

Commit 6c97fd8119

Commit `6c97fd8119`