Commit 8ddd998102

8ddd99810231b6941c71f65c27ddf1d48732e969

parent: db26399c9a

Verified · cmc ci/build: success ci/quality: success ci/test: success

cmc <hello@cleberg.net> · 2026-10-31 07:30 UTC

leaks: return at once when a scan stops early; specific keywords

readBlobs deferred cmd.Wait after cancel, so Wait ran first: an early
stop at MaxScanBytes or MaxFindings left git blocked writing output
nobody read, and the scan waited for its deadline. repo secret-scan on
krz/gitbay took the full two minutes; a large push would have been held
for the hook's ninety seconds. Cancel and Wait now run in one defer.

Rule keywords are now specific literals. "A" for the AWS rule and "gh",
"sk-", "shp" for others matched most files, which then ran expressions
that Go evaluates byte by byte.

Changelog for v1.46.1.

Layout: unified · split

CHANGELOG.org +12
@@ -4,6 +4,18 @@ Versioning follows semver from v0.1.0. Database migrations run
44automatically on daemon start; upgrade notes appear per release when
55anything beyond "replace the binary and restart" is needed.
66
7* v1.46.1 — 2026-10-31
8
9No migrations.
10
11- A secret scan that stops at its size or findings limit returns at
12 once. It used to wait for its deadline with git blocked on unread
13 output, so =repo secret-scan= on a large history took two minutes and
14 a push over the limit was held for ninety seconds.
15- The secret scan's rules are prefiltered on specific literals (=AKIA=,
16 =ghp_=, =sk-proj-=, …) rather than short ones that most source files
17 contain, which ran their expressions over nearly every file.
18
719* v1.46.0 — 2026-10-31
820
921Migrations 0080 and 0081. Every repository's pushes are secret-scanned
internal/leaks/leaks_test.go +30
@@ -7,6 +7,7 @@ import (
77 "path/filepath"
88 "strings"
99 "testing"
10 "time"
1011)
1112
1213// Token-shaped strings are assembled here rather than written out, so this
@@ -139,3 +140,32 @@ func TestAllowed(t *testing.T) {
139140 t.Errorf("allowed = %+v", got)
140141 }
141142}
143
144// Stopping part-way returns at once. git is left with more output than a
145// pipe holds, and Wait before cancel used to block until the deadline.
146func TestReadBlobsStopsPromptly(t *testing.T) {
147 dir := t.TempDir()
148 git(t, dir, "init", "-q", "-b", "main")
149 files := map[string]string{}
150 for i := 0; i < 20; i++ {
151 files[fake("f", 4, alnum)+string(rune('a'+i))] = strings.Repeat(string(rune('a'+i)), 200<<10)
152 }
153 commit(t, dir, files, "big")
154 oids, _, _, err := changedBlobs(context.Background(), dir, []string{"--all"})
155 if err != nil || len(oids) != 20 {
156 t.Fatalf("oids %d: %v", len(oids), err)
157 }
158 done := make(chan bool, 1)
159 go func() {
160 more, _ := readBlobs(context.Background(), dir, oids, func(string, []byte) bool { return false })
161 done <- more
162 }()
163 select {
164 case more := <-done:
165 if !more {
166 t.Error("an early stop was not reported")
167 }
168 case <-time.After(10 * time.Second):
169 t.Fatal("readBlobs did not return after fn stopped it")
170 }
171}
internal/leaks/rules.go +39 −25
@@ -10,34 +10,39 @@ import (
1010 "regexp"
1111)
1212
13// Rule is one credential format. keyword is a literal every match
14// contains, checked before the expression so most content costs a substring
15// search and nothing else.
13// Rule is one credential format. Every match contains one of keywords,
14// which are checked before the expression, so most content costs a few
15// substring searches and nothing else. They must be specific: Go's regexp
16// runs an expression that starts with \b byte by byte over the whole blob,
17// and a keyword common in source text (a bare "A") made a full-history scan
18// of this repository take minutes.
1619type Rule struct {
17 ID string
18 keyword string
19 re *regexp.Regexp
20 ID string
21 keywords []string
22 re *regexp.Regexp
2023}
2124
25func kw(k ...string) []string { return k }
26
2227var rules = []Rule{
23 {"private-key", "PRIVATE KEY", regexp.MustCompile(`-----BEGIN[ A-Z0-9]*PRIVATE KEY(?: BLOCK)?-----[ \t]*\r?\n(?:[A-Za-z0-9-]+: [^\n]*\r?\n)*\s*[A-Za-z0-9+/=]{40,}`)},
24 {"age-secret-key", "AGE-SECRET-KEY-1", regexp.MustCompile(`AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}`)},
25 {"aws-access-key-id", "A", regexp.MustCompile(`\b(?:AKIA|ASIA|ABIA|ACCA)[A-Z2-7]{16}\b`)},
26 {"github-token", "gh", regexp.MustCompile(`\bgh[pousr]_[A-Za-z0-9]{36}\b`)},
27 {"github-fine-grained-token", "github_pat_", regexp.MustCompile(`\bgithub_pat_[A-Za-z0-9_]{82}\b`)},
28 {"gitlab-token", "glpat-", regexp.MustCompile(`\bglpat-[A-Za-z0-9_\-]{20,}\b`)},
29 {"gitbay-token", "gb_", regexp.MustCompile(`\bgb_[0-9a-f]{64}\b`)},
30 {"slack-token", "xox", regexp.MustCompile(`\bxox[baprs]-[0-9]{6,}-[A-Za-z0-9-]{10,}\b`)},
31 {"slack-webhook", "hooks.slack.com", regexp.MustCompile(`https://hooks\.slack\.com/services/T[A-Z0-9]{6,}/B[A-Z0-9]{6,}/[A-Za-z0-9]{20,}`)},
32 {"stripe-live-key", "_live_", regexp.MustCompile(`\b[rs]k_live_[A-Za-z0-9]{24,}\b`)},
33 {"google-api-key", "AIza", regexp.MustCompile(`\bAIza[0-9A-Za-z_\-]{35}\b`)},
34 {"npm-token", "npm_", regexp.MustCompile(`\bnpm_[A-Za-z0-9]{36}\b`)},
35 {"pypi-token", "pypi-AgEIcHlwaS5vcmc", regexp.MustCompile(`\bpypi-AgEIcHlwaS5vcmc[A-Za-z0-9_\-]{50,}`)},
36 {"openai-key", "sk-", regexp.MustCompile(`\bsk-(?:proj|svcacct|admin)-[A-Za-z0-9_\-]{40,}`)},
37 {"anthropic-key", "sk-ant-", regexp.MustCompile(`\bsk-ant-(?:api|admin)[0-9]{2}-[A-Za-z0-9_\-]{80,}`)},
38 {"sendgrid-key", "SG.", regexp.MustCompile(`\bSG\.[A-Za-z0-9_\-]{22}\.[A-Za-z0-9_\-]{43}\b`)},
39 {"digitalocean-token", "_v1_", regexp.MustCompile(`\bdo[opr]_v1_[a-f0-9]{64}\b`)},
40 {"shopify-token", "shp", regexp.MustCompile(`\bshp(?:at|ca|pa|ss)_[a-fA-F0-9]{32}\b`)},
28 {"private-key", kw("PRIVATE KEY"), regexp.MustCompile(`-----BEGIN[ A-Z0-9]*PRIVATE KEY(?: BLOCK)?-----[ \t]*\r?\n(?:[A-Za-z0-9-]+: [^\n]*\r?\n)*\s*[A-Za-z0-9+/=]{40,}`)},
29 {"age-secret-key", kw("AGE-SECRET-KEY-1"), regexp.MustCompile(`AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}`)},
30 {"aws-access-key-id", kw("AKIA", "ASIA", "ABIA", "ACCA"), regexp.MustCompile(`\b(?:AKIA|ASIA|ABIA|ACCA)[A-Z2-7]{16}\b`)},
31 {"github-token", kw("ghp_", "gho_", "ghu_", "ghs_", "ghr_"), regexp.MustCompile(`\bgh[pousr]_[A-Za-z0-9]{36}\b`)},
32 {"github-fine-grained-token", kw("github_pat_"), regexp.MustCompile(`\bgithub_pat_[A-Za-z0-9_]{82}\b`)},
33 {"gitlab-token", kw("glpat-"), regexp.MustCompile(`\bglpat-[A-Za-z0-9_\-]{20,}\b`)},
34 {"gitbay-token", kw("gb_"), regexp.MustCompile(`\bgb_[0-9a-f]{64}\b`)},
35 {"slack-token", kw("xoxb-", "xoxa-", "xoxp-", "xoxr-", "xoxs-"), regexp.MustCompile(`\bxox[baprs]-[0-9]{6,}-[A-Za-z0-9-]{10,}\b`)},
36 {"slack-webhook", kw("hooks.slack.com/services/"), regexp.MustCompile(`https://hooks\.slack\.com/services/T[A-Z0-9]{6,}/B[A-Z0-9]{6,}/[A-Za-z0-9]{20,}`)},
37 {"stripe-live-key", kw("sk_live_", "rk_live_"), regexp.MustCompile(`\b[rs]k_live_[A-Za-z0-9]{24,}\b`)},
38 {"google-api-key", kw("AIza"), regexp.MustCompile(`\bAIza[0-9A-Za-z_\-]{35}\b`)},
39 {"npm-token", kw("npm_"), regexp.MustCompile(`\bnpm_[A-Za-z0-9]{36}\b`)},
40 {"pypi-token", kw("pypi-AgEIcHlwaS5vcmc"), regexp.MustCompile(`\bpypi-AgEIcHlwaS5vcmc[A-Za-z0-9_\-]{50,}`)},
41 {"openai-key", kw("sk-proj-", "sk-svcacct-", "sk-admin-"), regexp.MustCompile(`\bsk-(?:proj|svcacct|admin)-[A-Za-z0-9_\-]{40,}`)},
42 {"anthropic-key", kw("sk-ant-"), regexp.MustCompile(`\bsk-ant-(?:api|admin)[0-9]{2}-[A-Za-z0-9_\-]{80,}`)},
43 {"sendgrid-key", kw("SG."), regexp.MustCompile(`\bSG\.[A-Za-z0-9_\-]{22}\.[A-Za-z0-9_\-]{43}\b`)},
44 {"digitalocean-token", kw("dop_v1_", "doo_v1_", "dor_v1_"), regexp.MustCompile(`\bdo[opr]_v1_[a-f0-9]{64}\b`)},
45 {"shopify-token", kw("shpat_", "shpca_", "shppa_", "shpss_"), regexp.MustCompile(`\bshp(?:at|ca|pa|ss)_[a-fA-F0-9]{32}\b`)},
4146}
4247
4348// Hit is one match inside a blob.
@@ -54,7 +59,7 @@ const maxHitsPerBlob = 10
5459func Find(data []byte) []Hit {
5560 var out []Hit
5661 for _, r := range rules {
57 if !bytes.Contains(data, []byte(r.keyword)) {
62 if !containsAny(data, r.keywords) {
5863 continue
5964 }
6065 last := 0
@@ -78,3 +83,12 @@ func Find(data []byte) []Hit {
7883func binary(data []byte) bool {
7984 return bytes.IndexByte(data[:min(len(data), 8000)], 0) >= 0
8085}
86
87func containsAny(data []byte, keywords []string) bool {
88 for _, k := range keywords {
89 if bytes.Contains(data, []byte(k)) {
90 return true
91 }
92 }
93 return false
94}
internal/leaks/scan.go +7 −1
@@ -177,7 +177,13 @@ func readBlobs(ctx context.Context, dir string, oids []string, fn func(oid strin
177177 if err := cmd.Start(); err != nil {
178178 return false, fmt.Errorf("cat-file --batch: %w", err)
179179 }
180 defer cmd.Wait() // after cancel, which ends git on an early return
180 // Cancel before Wait, in one defer: on an early return git is blocked
181 // writing output nobody reads, and Wait alone would wait for the
182 // deadline to kill it.
183 defer func() {
184 cancel()
185 cmd.Wait()
186 }()
181187 // Fed from another goroutine: the pipe holds fewer names than a large
182188 // push has, and git blocks writing output nobody is reading yet.
183189 go func() {