Commit 359b4a6fb4
Verified · cmc
Layout: unified · split
internal/codescan/codescan.go +113 −19
| @@ -12,6 +12,7 @@ import ( | ||
| 12 | 12 | "fmt" |
| 13 | 13 | "net/url" |
| 14 | 14 | "sort" |
| 15 | "strconv" | |
| 15 | 16 | "strings" |
| 16 | 17 | |
| 17 | 18 | "gitbay.org/gitbay/internal/store" |
| @@ -46,9 +47,14 @@ type sarif struct { | ||
| 46 | 47 | } `json:"artifactLocation"` |
| 47 | 48 | Region struct { |
| 48 | 49 | StartLine int `json:"startLine"` |
| 50 | Snippet struct { | |
| 51 | Text string `json:"text"` | |
| 52 | } `json:"snippet"` | |
| 49 | 53 | } `json:"region"` |
| 50 | 54 | } `json:"physicalLocation"` |
| 51 | 55 | } `json:"locations"` |
| 56 | Fingerprints map[string]string `json:"fingerprints"` | |
| 57 | PartialFingerprints map[string]string `json:"partialFingerprints"` | |
| 52 | 58 | } `json:"results"` |
| 53 | 59 | } `json:"runs"` |
| 54 | 60 | } |
| @@ -64,30 +70,52 @@ func Parse(raw []byte) ([]store.SecurityFinding, bool, error) { | ||
| 64 | 70 | if doc.Runs == nil { |
| 65 | 71 | return nil, false, errors.New("not SARIF: no runs") |
| 66 | 72 | } |
| 67 | seen := map[string]bool{} | |
| 68 | var out []store.SecurityFinding | |
| 69 | truncated := false | |
| 73 | var hits []hit | |
| 70 | 74 | for _, run := range doc.Runs { |
| 71 | 75 | tool := clean(run.Tool.Driver.Name, 40) |
| 76 | var found []hit | |
| 72 | 77 | for _, r := range run.Results { |
| 73 | if len(out) >= MaxFindings { | |
| 74 | truncated = true | |
| 75 | break | |
| 76 | } | |
| 77 | f := store.SecurityFinding{Tool: tool, Rule: clean(r.RuleID, 200), | |
| 78 | Level: level(r.Level), Message: clean(r.Message.Text, 200)} | |
| 78 | h := hit{f: store.SecurityFinding{Tool: tool, Rule: clean(r.RuleID, 200), | |
| 79 | Level: level(r.Level), Message: clean(r.Message.Text, 200)}} | |
| 79 | 80 | if len(r.Locations) > 0 { |
| 80 | 81 | loc := r.Locations[0].PhysicalLocation |
| 81 | f.Path = relPath(loc.ArtifactLocation.URI) | |
| 82 | f.Line = max(loc.Region.StartLine, 0) | |
| 82 | h.f.Path = relPath(loc.ArtifactLocation.URI) | |
| 83 | h.f.Line = max(loc.Region.StartLine, 0) | |
| 84 | h.snippet = strings.Join(strings.Fields(loc.Region.Snippet.Text), " ") | |
| 83 | 85 | } |
| 84 | f.Fingerprint = fingerprint(f) | |
| 85 | if seen[f.Fingerprint] { | |
| 86 | continue | |
| 86 | h.own = ownFingerprint(r.Fingerprints, r.PartialFingerprints) | |
| 87 | found = append(found, h) | |
| 88 | } | |
| 89 | if !distinct(found) { | |
| 90 | for i := range found { | |
| 91 | found[i].own = "" | |
| 87 | 92 | } |
| 88 | seen[f.Fingerprint] = true | |
| 89 | out = append(out, f) | |
| 90 | 93 | } |
| 94 | hits = append(hits, found...) | |
| 95 | } | |
| 96 | // File order, so hits that look alike are numbered the same way on | |
| 97 | // every scan. | |
| 98 | sort.SliceStable(hits, func(i, j int) bool { | |
| 99 | if hits[i].f.Path != hits[j].f.Path { | |
| 100 | return hits[i].f.Path < hits[j].f.Path | |
| 101 | } | |
| 102 | return hits[i].f.Line < hits[j].f.Line | |
| 103 | }) | |
| 104 | seen := map[string]bool{} | |
| 105 | alike := map[string]int{} | |
| 106 | var out []store.SecurityFinding | |
| 107 | truncated := false | |
| 108 | for _, h := range hits { | |
| 109 | if seen[h.place()] { | |
| 110 | continue | |
| 111 | } | |
| 112 | seen[h.place()] = true | |
| 113 | if len(out) >= MaxFindings { | |
| 114 | truncated = true | |
| 115 | break | |
| 116 | } | |
| 117 | h.f.Fingerprint = fingerprint(h, alike) | |
| 118 | out = append(out, h.f) | |
| 91 | 119 | } |
| 92 | 120 | sort.Slice(out, func(i, j int) bool { |
| 93 | 121 | if out[i].Path != out[j].Path { |
| @@ -101,10 +129,76 @@ func Parse(raw []byte) ([]store.SecurityFinding, bool, error) { | ||
| 101 | 129 | return out, truncated, nil |
| 102 | 130 | } |
| 103 | 131 | |
| 132 | // hit is one result before its fingerprint is settled: own is the tool's | |
| 133 | // fingerprint for it, snippet the code it matched, either possibly empty. | |
| 134 | type hit struct { | |
| 135 | f store.SecurityFinding | |
| 136 | own, snippet string | |
| 137 | } | |
| 138 | ||
| 139 | // place is where a result was reported. One reported twice at the same | |
| 140 | // place is one finding. | |
| 141 | func (h hit) place() string { | |
| 142 | return h.f.Tool + "\x00" + h.f.Rule + "\x00" + h.f.Path + "\x00" + strconv.Itoa(h.f.Line) + "\x00" + h.f.Message | |
| 143 | } | |
| 144 | ||
| 145 | // ownFingerprint picks the tool's fingerprint for a result, preferring the | |
| 146 | // full ones over partial, and the first by name for a stable choice. | |
| 147 | func ownFingerprint(sets ...map[string]string) string { | |
| 148 | for _, set := range sets { | |
| 149 | names := make([]string, 0, len(set)) | |
| 150 | for name := range set { | |
| 151 | names = append(names, name) | |
| 152 | } | |
| 153 | sort.Strings(names) | |
| 154 | for _, name := range names { | |
| 155 | if v := set[name]; v != "" && !strings.ContainsAny(v, " \t\r\n") && len(v) <= 200 { | |
| 156 | return name + "=" + v | |
| 157 | } | |
| 158 | } | |
| 159 | } | |
| 160 | return "" | |
| 161 | } | |
| 162 | ||
| 163 | // distinct reports whether a run's own fingerprints can be trusted: every | |
| 164 | // result has one, and no two places share one. A tool that writes a | |
| 165 | // placeholder or a per-rule constant there falls back to ours. | |
| 166 | func distinct(found []hit) bool { | |
| 167 | at := map[string]string{} | |
| 168 | for _, h := range found { | |
| 169 | if h.own == "" { | |
| 170 | return false | |
| 171 | } | |
| 172 | if p, ok := at[h.own]; ok && p != h.place() { | |
| 173 | return false | |
| 174 | } | |
| 175 | at[h.own] = h.place() | |
| 176 | } | |
| 177 | return true | |
| 178 | } | |
| 179 | ||
| 104 | 180 | // fingerprint identifies a finding across scans. The line is left out so |
| 105 | // an edit above it does not make it a new finding. | |
| 106 | func fingerprint(f store.SecurityFinding) string { | |
| 107 | sum := sha256.Sum256([]byte(f.Tool + "\x00" + f.Rule + "\x00" + f.Path + "\x00" + f.Message)) | |
| 181 | // an edit above it does not make it a new finding. The tool's own | |
| 182 | // fingerprint is used when it gives one. Otherwise the matched code tells | |
| 183 | // apart hits of one rule in one file, and hits that still look alike are | |
| 184 | // numbered in file order, the first keeping the plain form; alike counts | |
| 185 | // them across calls. | |
| 186 | func fingerprint(h hit, alike map[string]int) string { | |
| 187 | key := h.f.Tool + "\x00" + h.f.Rule + "\x00" + h.f.Path + "\x00" | |
| 188 | if h.own != "" { | |
| 189 | key += "own\x00" + h.own | |
| 190 | } else { | |
| 191 | key += h.f.Message | |
| 192 | if h.snippet != "" { | |
| 193 | key += "\x00" + h.snippet | |
| 194 | } | |
| 195 | } | |
| 196 | n := alike[key] | |
| 197 | alike[key]++ | |
| 198 | if n > 0 { | |
| 199 | key += "\x00" + strconv.Itoa(n) | |
| 200 | } | |
| 201 | sum := sha256.Sum256([]byte(key)) | |
| 108 | 202 | return hex.EncodeToString(sum[:12]) |
| 109 | 203 | } |
| 110 | 204 | |
internal/codescan/codescan_test.go +74
| @@ -1,6 +1,7 @@ | ||
| 1 | 1 | package codescan |
| 2 | 2 | |
| 3 | 3 | import ( |
| 4 | "gitbay.org/gitbay/internal/store" | |
| 4 | 5 | "strconv" |
| 5 | 6 | "strings" |
| 6 | 7 | "testing" |
| @@ -69,3 +70,76 @@ func TestSameIgnoresLines(t *testing.T) { | ||
| 69 | 70 | t.Error("a moved finding counted as a change") |
| 70 | 71 | } |
| 71 | 72 | } |
| 73 | ||
| 74 | // sarifOf builds a one-run document from results given as JSON objects. | |
| 75 | func sarifOf(results ...string) []byte { | |
| 76 | return []byte(`{"runs":[{"tool":{"driver":{"name":"t"}},"results":[` + strings.Join(results, ",") + `]}]}`) | |
| 77 | } | |
| 78 | ||
| 79 | func result(line int, snippet, extra string) string { | |
| 80 | region := `"startLine":` + itoa(line) | |
| 81 | if snippet != "" { | |
| 82 | region += `,"snippet":{"text":` + strconv.Quote(snippet) + `}` | |
| 83 | } | |
| 84 | if extra != "" { | |
| 85 | extra = "," + extra | |
| 86 | } | |
| 87 | return `{"ruleId":"r","message":{"text":"m"},"locations":[{"physicalLocation":{"artifactLocation":{"uri":"x.go"},"region":{` + | |
| 88 | region + `}}}]` + extra + `}` | |
| 89 | } | |
| 90 | ||
| 91 | func fingerprints(t *testing.T, raw []byte) []string { | |
| 92 | t.Helper() | |
| 93 | got, _, err := Parse(raw) | |
| 94 | if err != nil { | |
| 95 | t.Fatal(err) | |
| 96 | } | |
| 97 | var out []string | |
| 98 | for _, f := range got { | |
| 99 | out = append(out, f.Fingerprint) | |
| 100 | } | |
| 101 | return out | |
| 102 | } | |
| 103 | ||
| 104 | // Hits of one rule in one file are separate findings, and the first keeps | |
| 105 | // the fingerprint a scan without snippets always gave it. | |
| 106 | func TestParseKeepsAlikeHits(t *testing.T) { | |
| 107 | fps := fingerprints(t, sarifOf(result(3, "", ""), result(9, "", ""))) | |
| 108 | if len(fps) != 2 || fps[0] == fps[1] { | |
| 109 | t.Fatalf("fingerprints = %v", fps) | |
| 110 | } | |
| 111 | if want := fingerprint(hit{f: store.SecurityFinding{Tool: "t", Rule: "r", Path: "x.go", Message: "m"}}, map[string]int{}); fps[0] != want { | |
| 112 | t.Errorf("first hit = %s, want %s", fps[0], want) | |
| 113 | } | |
| 114 | // Moved, they are still the same two. | |
| 115 | if moved := fingerprints(t, sarifOf(result(13, "", ""), result(19, "", ""))); strings.Join(moved, " ") != strings.Join(fps, " ") { | |
| 116 | t.Errorf("moved = %v, was %v", moved, fps) | |
| 117 | } | |
| 118 | } | |
| 119 | ||
| 120 | // With the matched code, a new hit above others leaves theirs alone, and | |
| 121 | // reindenting the code changes nothing. | |
| 122 | func TestParseSnippetsSurviveInsertion(t *testing.T) { | |
| 123 | before := fingerprints(t, sarifOf(result(3, "a()", ""), result(9, "b(x, y)", ""))) | |
| 124 | after := fingerprints(t, sarifOf(result(1, "c()", ""), result(4, "a()", ""), result(10, "\t b(x,\n y)", ""))) | |
| 125 | if len(after) != 3 || after[1] != before[0] || after[2] != before[1] { | |
| 126 | t.Errorf("before %v after %v", before, after) | |
| 127 | } | |
| 128 | } | |
| 129 | ||
| 130 | // The tool's own fingerprints are used when every result has a distinct | |
| 131 | // one, and ignored when they are placeholders or shared. | |
| 132 | func TestParseOwnFingerprints(t *testing.T) { | |
| 133 | own := func(v string) string { return `"fingerprints":{"matchBasedId/v1":"` + v + `"}` } | |
| 134 | a := fingerprints(t, sarifOf(result(3, "", own("aaa")), result(9, "", own("bbb")))) | |
| 135 | b := fingerprints(t, sarifOf(result(30, "", own("aaa")), result(1, "", own("bbb")))) | |
| 136 | if len(a) != 2 || a[0] != b[1] || a[1] != b[0] { | |
| 137 | t.Errorf("own fingerprints not followed: %v %v", a, b) | |
| 138 | } | |
| 139 | plain := fingerprints(t, sarifOf(result(3, "", ""), result(9, "", ""))) | |
| 140 | for _, v := range []string{"requires login", "same"} { | |
| 141 | if got := fingerprints(t, sarifOf(result(3, "", own(v)), result(9, "", own(v)))); strings.Join(got, " ") != strings.Join(plain, " ") { | |
| 142 | t.Errorf("%q: got %v, want the fallback %v", v, got, plain) | |
| 143 | } | |
| 144 | } | |
| 145 | } | |