Commit 359b4a6fb4

359b4a6fb49bc8d6fa5c8baf17ea1b48e8db0507

parent: 56c8b81ae6

Verified · cmc

cmc <hello@cleberg.net> · 2026-11-01 05:53 UTC

codescan: keep hits of one rule in one file apart

The fingerprint was tool, rule, path and message, so every hit of a rule
in a file with the same message folded into one finding. Use the tool's
own fingerprint when every result in the run has a distinct one, else the
matched snippet, and number hits that still look alike in file order. The
first keeps the old fingerprint.

Ref #344

Layout: unified · split

internal/codescan/codescan.go +113 −19
@@ -12,6 +12,7 @@ import (
1212 "fmt"
1313 "net/url"
1414 "sort"
15 "strconv"
1516 "strings"
1617
1718 "gitbay.org/gitbay/internal/store"
@@ -46,9 +47,14 @@ type sarif struct {
4647 } `json:"artifactLocation"`
4748 Region struct {
4849 StartLine int `json:"startLine"`
50 Snippet struct {
51 Text string `json:"text"`
52 } `json:"snippet"`
4953 } `json:"region"`
5054 } `json:"physicalLocation"`
5155 } `json:"locations"`
56 Fingerprints map[string]string `json:"fingerprints"`
57 PartialFingerprints map[string]string `json:"partialFingerprints"`
5258 } `json:"results"`
5359 } `json:"runs"`
5460}
@@ -64,30 +70,52 @@ func Parse(raw []byte) ([]store.SecurityFinding, bool, error) {
6470 if doc.Runs == nil {
6571 return nil, false, errors.New("not SARIF: no runs")
6672 }
67 seen := map[string]bool{}
68 var out []store.SecurityFinding
69 truncated := false
73 var hits []hit
7074 for _, run := range doc.Runs {
7175 tool := clean(run.Tool.Driver.Name, 40)
76 var found []hit
7277 for _, r := range run.Results {
73 if len(out) >= MaxFindings {
74 truncated = true
75 break
76 }
77 f := store.SecurityFinding{Tool: tool, Rule: clean(r.RuleID, 200),
78 Level: level(r.Level), Message: clean(r.Message.Text, 200)}
78 h := hit{f: store.SecurityFinding{Tool: tool, Rule: clean(r.RuleID, 200),
79 Level: level(r.Level), Message: clean(r.Message.Text, 200)}}
7980 if len(r.Locations) > 0 {
8081 loc := r.Locations[0].PhysicalLocation
81 f.Path = relPath(loc.ArtifactLocation.URI)
82 f.Line = max(loc.Region.StartLine, 0)
82 h.f.Path = relPath(loc.ArtifactLocation.URI)
83 h.f.Line = max(loc.Region.StartLine, 0)
84 h.snippet = strings.Join(strings.Fields(loc.Region.Snippet.Text), " ")
8385 }
84 f.Fingerprint = fingerprint(f)
85 if seen[f.Fingerprint] {
86 continue
86 h.own = ownFingerprint(r.Fingerprints, r.PartialFingerprints)
87 found = append(found, h)
88 }
89 if !distinct(found) {
90 for i := range found {
91 found[i].own = ""
8792 }
88 seen[f.Fingerprint] = true
89 out = append(out, f)
9093 }
94 hits = append(hits, found...)
95 }
96 // File order, so hits that look alike are numbered the same way on
97 // every scan.
98 sort.SliceStable(hits, func(i, j int) bool {
99 if hits[i].f.Path != hits[j].f.Path {
100 return hits[i].f.Path < hits[j].f.Path
101 }
102 return hits[i].f.Line < hits[j].f.Line
103 })
104 seen := map[string]bool{}
105 alike := map[string]int{}
106 var out []store.SecurityFinding
107 truncated := false
108 for _, h := range hits {
109 if seen[h.place()] {
110 continue
111 }
112 seen[h.place()] = true
113 if len(out) >= MaxFindings {
114 truncated = true
115 break
116 }
117 h.f.Fingerprint = fingerprint(h, alike)
118 out = append(out, h.f)
91119 }
92120 sort.Slice(out, func(i, j int) bool {
93121 if out[i].Path != out[j].Path {
@@ -101,10 +129,76 @@ func Parse(raw []byte) ([]store.SecurityFinding, bool, error) {
101129 return out, truncated, nil
102130}
103131
132// hit is one result before its fingerprint is settled: own is the tool's
133// fingerprint for it, snippet the code it matched, either possibly empty.
134type hit struct {
135 f store.SecurityFinding
136 own, snippet string
137}
138
139// place is where a result was reported. One reported twice at the same
140// place is one finding.
141func (h hit) place() string {
142 return h.f.Tool + "\x00" + h.f.Rule + "\x00" + h.f.Path + "\x00" + strconv.Itoa(h.f.Line) + "\x00" + h.f.Message
143}
144
145// ownFingerprint picks the tool's fingerprint for a result, preferring the
146// full ones over partial, and the first by name for a stable choice.
147func ownFingerprint(sets ...map[string]string) string {
148 for _, set := range sets {
149 names := make([]string, 0, len(set))
150 for name := range set {
151 names = append(names, name)
152 }
153 sort.Strings(names)
154 for _, name := range names {
155 if v := set[name]; v != "" && !strings.ContainsAny(v, " \t\r\n") && len(v) <= 200 {
156 return name + "=" + v
157 }
158 }
159 }
160 return ""
161}
162
163// distinct reports whether a run's own fingerprints can be trusted: every
164// result has one, and no two places share one. A tool that writes a
165// placeholder or a per-rule constant there falls back to ours.
166func distinct(found []hit) bool {
167 at := map[string]string{}
168 for _, h := range found {
169 if h.own == "" {
170 return false
171 }
172 if p, ok := at[h.own]; ok && p != h.place() {
173 return false
174 }
175 at[h.own] = h.place()
176 }
177 return true
178}
179
104180// fingerprint identifies a finding across scans. The line is left out so
105// an edit above it does not make it a new finding.
106func fingerprint(f store.SecurityFinding) string {
107 sum := sha256.Sum256([]byte(f.Tool + "\x00" + f.Rule + "\x00" + f.Path + "\x00" + f.Message))
181// an edit above it does not make it a new finding. The tool's own
182// fingerprint is used when it gives one. Otherwise the matched code tells
183// apart hits of one rule in one file, and hits that still look alike are
184// numbered in file order, the first keeping the plain form; alike counts
185// them across calls.
186func fingerprint(h hit, alike map[string]int) string {
187 key := h.f.Tool + "\x00" + h.f.Rule + "\x00" + h.f.Path + "\x00"
188 if h.own != "" {
189 key += "own\x00" + h.own
190 } else {
191 key += h.f.Message
192 if h.snippet != "" {
193 key += "\x00" + h.snippet
194 }
195 }
196 n := alike[key]
197 alike[key]++
198 if n > 0 {
199 key += "\x00" + strconv.Itoa(n)
200 }
201 sum := sha256.Sum256([]byte(key))
108202 return hex.EncodeToString(sum[:12])
109203}
110204
internal/codescan/codescan_test.go +74
@@ -1,6 +1,7 @@
11package codescan
22
33import (
4 "gitbay.org/gitbay/internal/store"
45 "strconv"
56 "strings"
67 "testing"
@@ -69,3 +70,76 @@ func TestSameIgnoresLines(t *testing.T) {
6970 t.Error("a moved finding counted as a change")
7071 }
7172}
73
74// sarifOf builds a one-run document from results given as JSON objects.
75func sarifOf(results ...string) []byte {
76 return []byte(`{"runs":[{"tool":{"driver":{"name":"t"}},"results":[` + strings.Join(results, ",") + `]}]}`)
77}
78
79func result(line int, snippet, extra string) string {
80 region := `"startLine":` + itoa(line)
81 if snippet != "" {
82 region += `,"snippet":{"text":` + strconv.Quote(snippet) + `}`
83 }
84 if extra != "" {
85 extra = "," + extra
86 }
87 return `{"ruleId":"r","message":{"text":"m"},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"x.go"},"region":{` +
88 region + `}}}]` + extra + `}`
89}
90
91func fingerprints(t *testing.T, raw []byte) []string {
92 t.Helper()
93 got, _, err := Parse(raw)
94 if err != nil {
95 t.Fatal(err)
96 }
97 var out []string
98 for _, f := range got {
99 out = append(out, f.Fingerprint)
100 }
101 return out
102}
103
104// Hits of one rule in one file are separate findings, and the first keeps
105// the fingerprint a scan without snippets always gave it.
106func TestParseKeepsAlikeHits(t *testing.T) {
107 fps := fingerprints(t, sarifOf(result(3, "", ""), result(9, "", "")))
108 if len(fps) != 2 || fps[0] == fps[1] {
109 t.Fatalf("fingerprints = %v", fps)
110 }
111 if want := fingerprint(hit{f: store.SecurityFinding{Tool: "t", Rule: "r", Path: "x.go", Message: "m"}}, map[string]int{}); fps[0] != want {
112 t.Errorf("first hit = %s, want %s", fps[0], want)
113 }
114 // Moved, they are still the same two.
115 if moved := fingerprints(t, sarifOf(result(13, "", ""), result(19, "", ""))); strings.Join(moved, " ") != strings.Join(fps, " ") {
116 t.Errorf("moved = %v, was %v", moved, fps)
117 }
118}
119
120// With the matched code, a new hit above others leaves theirs alone, and
121// reindenting the code changes nothing.
122func TestParseSnippetsSurviveInsertion(t *testing.T) {
123 before := fingerprints(t, sarifOf(result(3, "a()", ""), result(9, "b(x, y)", "")))
124 after := fingerprints(t, sarifOf(result(1, "c()", ""), result(4, "a()", ""), result(10, "\t b(x,\n y)", "")))
125 if len(after) != 3 || after[1] != before[0] || after[2] != before[1] {
126 t.Errorf("before %v after %v", before, after)
127 }
128}
129
130// The tool's own fingerprints are used when every result has a distinct
131// one, and ignored when they are placeholders or shared.
132func TestParseOwnFingerprints(t *testing.T) {
133 own := func(v string) string { return `"fingerprints":{"matchBasedId/v1":"` + v + `"}` }
134 a := fingerprints(t, sarifOf(result(3, "", own("aaa")), result(9, "", own("bbb"))))
135 b := fingerprints(t, sarifOf(result(30, "", own("aaa")), result(1, "", own("bbb"))))
136 if len(a) != 2 || a[0] != b[1] || a[1] != b[0] {
137 t.Errorf("own fingerprints not followed: %v %v", a, b)
138 }
139 plain := fingerprints(t, sarifOf(result(3, "", ""), result(9, "", "")))
140 for _, v := range []string{"requires login", "same"} {
141 if got := fingerprints(t, sarifOf(result(3, "", own(v)), result(9, "", own(v)))); strings.Join(got, " ") != strings.Join(plain, " ") {
142 t.Errorf("%q: got %v, want the fallback %v", v, got, plain)
143 }
144 }
145}