internal/store/builds.go
324 lines · 10196 bytes
1package store
2
3import (
4 "database/sql"
5 "errors"
6 "os"
7 "strconv"
8 "strings"
9 "time"
10)
11
12// Build is one CI job execution for one commit.
13type Build struct {
14 ID int64
15 RepoID int64
16 Number int64
17 Job string
18 SHA string
19 Ref string
20 Steps string // JSON array of shell commands
21 Status string // pending|running|success|failure
22 CreatedAt string
23 StartedAt string
24 FinishedAt string
25 // Trusted is false for a merge request head fetched from another
26 // repository: its steps run without the target's secrets.
27 Trusted bool
28}
29
30// MaxBuildLog caps a build's stored log; appends past it are dropped.
31const MaxBuildLog = 2 << 20
32
33// truncNotice is appended once when a log first hits the cap. A log that
34// simply stops is indistinguishable from a build that died mid-step, which
35// is the reading that sent people hunting for a nonexistent test failure.
36var truncNotice = []byte("\n[log truncated: reached the " +
37 strconv.Itoa(MaxBuildLog>>20) + " MiB cap; earlier output is above]\n")
38
39// CreateBuild allocates the per-repo build number in the same transaction
40// as the insert, like issue and MR numbers.
41func (s *Store) CreateBuild(repoID int64, job, sha, ref, stepsJSON string, trusted bool) (int64, error) {
42 tx, err := s.DB.Begin()
43 if err != nil {
44 return 0, err
45 }
46 defer tx.Rollback()
47 if _, err := tx.Exec("UPDATE repos SET build_counter = build_counter + 1 WHERE id = ?", repoID); err != nil {
48 return 0, err
49 }
50 var n int64
51 if err := tx.QueryRow("SELECT build_counter FROM repos WHERE id = ?", repoID).Scan(&n); err != nil {
52 return 0, err
53 }
54 if _, err := tx.Exec(
55 "INSERT INTO builds (repo_id, number, job, sha, ref, steps, trusted) VALUES (?, ?, ?, ?, ?, ?, ?)",
56 repoID, n, job, sha, ref, stepsJSON, trusted); err != nil {
57 return 0, err
58 }
59 return n, tx.Commit()
60}
61
62const buildSelect = `
63 SELECT id, repo_id, number, job, sha, ref, steps, status, created_at, started_at, finished_at, trusted
64 FROM builds`
65
66func scanBuild(row interface{ Scan(...any) error }) (Build, error) {
67 var b Build
68 var trusted int
69 err := row.Scan(&b.ID, &b.RepoID, &b.Number, &b.Job, &b.SHA, &b.Ref, &b.Steps,
70 &b.Status, &b.CreatedAt, &b.StartedAt, &b.FinishedAt, &trusted)
71 b.Trusted = trusted != 0
72 return b, err
73}
74
75// ClaimBuild atomically hands the oldest pending build to a runner.
76// ClaimBuild takes the oldest pending build and marks it running. A
77// non-empty repoIDs restricts the claim to those repositories, which is how
78// a runner on a machine that should not execute every repository's steps
79// limits what it picks up.
80func (s *Store) ClaimBuild(repoIDs []int64) (Build, bool, error) {
81 tx, err := s.DB.Begin()
82 if err != nil {
83 return Build{}, false, err
84 }
85 defer tx.Rollback()
86 query := "SELECT id FROM builds WHERE status = 'pending' ORDER BY id LIMIT 1"
87 args := []any{}
88 if len(repoIDs) > 0 {
89 marks := strings.TrimSuffix(strings.Repeat("?,", len(repoIDs)), ",")
90 query = "SELECT id FROM builds WHERE status = 'pending' AND repo_id IN (" +
91 marks + ") ORDER BY id LIMIT 1"
92 for _, id := range repoIDs {
93 args = append(args, id)
94 }
95 }
96 var id int64
97 err = tx.QueryRow(query, args...).Scan(&id)
98 if errors.Is(err, sql.ErrNoRows) {
99 return Build{}, false, nil
100 }
101 if err != nil {
102 return Build{}, false, err
103 }
104 if _, err := tx.Exec(
105 "UPDATE builds SET status = 'running', started_at = strftime('%Y-%m-%dT%H:%M:%SZ','now') WHERE id = ?", id); err != nil {
106 return Build{}, false, err
107 }
108 b, err := scanBuild(tx.QueryRow(buildSelect+" WHERE id = ?", id))
109 if err != nil {
110 return Build{}, false, err
111 }
112 return b, true, tx.Commit()
113}
114
115// StaleBuildDeadline is how long a claimed build may stay running before the
116// server gives up on it. Comfortably longer than the runner's own -timeout
117// (45m by default), so this only fires when the runner never reported at all —
118// it was killed, restarted, or lost the network mid-build.
119const StaleBuildDeadline = 90 * time.Minute
120
121// staleBuildDeadline is StaleBuildDeadline unless GITBAY_STALE_BUILD_DEADLINE
122// shortens it, which tests do.
123func staleBuildDeadline() time.Duration {
124 if v := os.Getenv("GITBAY_STALE_BUILD_DEADLINE"); v != "" {
125 if d, err := time.ParseDuration(v); err == nil && d > 0 {
126 return d
127 }
128 }
129 return StaleBuildDeadline
130}
131
132// ReapStaleBuilds fails every build that has been running past the deadline and
133// returns them, so the caller can resolve their commit statuses. A runner that
134// dies between claiming a build and reporting it otherwise leaves the row
135// claimed forever, and the commit pending forever with it.
136func (s *Store) ReapStaleBuilds() ([]Build, error) {
137 cutoff := time.Now().UTC().Add(-staleBuildDeadline()).Format("2006-01-02T15:04:05Z")
138 rows, err := s.DB.Query(buildSelect+
139 " WHERE status = 'running' AND started_at != '' AND started_at < ?", cutoff)
140 if err != nil {
141 return nil, err
142 }
143 defer rows.Close()
144 var stale []Build
145 for rows.Next() {
146 b, err := scanBuild(rows)
147 if err != nil {
148 return nil, err
149 }
150 stale = append(stale, b)
151 }
152 if err := rows.Err(); err != nil {
153 return nil, err
154 }
155 for _, b := range stale {
156 if err := s.AppendBuildLog(b.ID, []byte(
157 "\nbuild abandoned: the runner never reported an outcome\n")); err != nil {
158 return nil, err
159 }
160 if err := s.FinishBuild(b.ID, "failure"); err != nil {
161 return nil, err
162 }
163 }
164 return stale, nil
165}
166
167// AppendBuildLog adds a chunk to the build's log, dropping bytes past the cap.
168func (s *Store) AppendBuildLog(id int64, chunk []byte) error {
169 res, err := s.DB.Exec(`
170 UPDATE builds SET log = log || ?
171 WHERE id = ? AND length(log) < ?`, chunk, id, MaxBuildLog)
172 if err != nil {
173 return err
174 }
175 if n, _ := res.RowsAffected(); n > 0 {
176 return nil
177 }
178 // Over the cap. The bounds match exactly once: appending the notice puts
179 // the log past the upper bound, so later chunks fall through silently.
180 _, err = s.DB.Exec(`
181 UPDATE builds SET log = log || ?
182 WHERE id = ? AND length(log) >= ? AND length(log) < ?`,
183 truncNotice, id, MaxBuildLog, MaxBuildLog+len(truncNotice))
184 return err
185}
186
187// FinishBuild records the outcome of a running build.
188func (s *Store) FinishBuild(id int64, status string) error {
189 res, err := s.DB.Exec(`
190 UPDATE builds SET status = ?, finished_at = strftime('%Y-%m-%dT%H:%M:%SZ','now')
191 WHERE id = ? AND status = 'running'`, status, id)
192 if err != nil {
193 return err
194 }
195 if n, _ := res.RowsAffected(); n == 0 {
196 return ErrNotFound
197 }
198 return nil
199}
200
201func (s *Store) BuildByID(id int64) (Build, error) {
202 b, err := scanBuild(s.DB.QueryRow(buildSelect+" WHERE id = ?", id))
203 if errors.Is(err, sql.ErrNoRows) {
204 return b, ErrNotFound
205 }
206 return b, err
207}
208
209func (s *Store) BuildByNumber(repoID, number int64) (Build, error) {
210 b, err := scanBuild(s.DB.QueryRow(buildSelect+" WHERE repo_id = ? AND number = ?", repoID, number))
211 if errors.Is(err, sql.ErrNoRows) {
212 return b, ErrNotFound
213 }
214 return b, err
215}
216
217func (s *Store) ListBuilds(repoID int64, limit int) ([]Build, error) {
218 rows, err := s.DB.Query(buildSelect+" WHERE repo_id = ? ORDER BY number DESC LIMIT ?", repoID, limit)
219 if err != nil {
220 return nil, err
221 }
222 defer rows.Close()
223 var out []Build
224 for rows.Next() {
225 b, err := scanBuild(rows)
226 if err != nil {
227 return nil, err
228 }
229 out = append(out, b)
230 }
231 return out, rows.Err()
232}
233
234// BuildLog returns the stored log bytes.
235func (s *Store) BuildLog(id int64) ([]byte, error) {
236 var log []byte
237 err := s.DB.QueryRow("SELECT log FROM builds WHERE id = ?", id).Scan(&log)
238 if errors.Is(err, sql.ErrNoRows) {
239 return nil, ErrNotFound
240 }
241 return log, err
242}
243
244// LatestBuild returns the newest build for a repo, optionally narrowed to
245// one job. It is what a status badge reports.
246func (s *Store) LatestBuild(repoID int64, job string) (Build, error) {
247 q := buildSelect + " WHERE repo_id = ?"
248 args := []any{repoID}
249 if job != "" {
250 q += " AND job = ?"
251 args = append(args, job)
252 }
253 q += " ORDER BY number DESC LIMIT 1"
254 b, err := scanBuild(s.DB.QueryRow(q, args...))
255 if errors.Is(err, sql.ErrNoRows) {
256 return b, ErrNotFound
257 }
258 return b, err
259}
260
261// BuildsForCommit returns the newest build per job for one commit. A merge
262// request's checks are ci/<job> statuses; this is where their timing comes
263// from, in one query rather than one per check.
264func (s *Store) BuildsForCommit(repoID int64, sha string) (map[string]Build, error) {
265 rows, err := s.DB.Query(buildSelect+" WHERE repo_id = ? AND sha = ? ORDER BY number ASC", repoID, sha)
266 if err != nil {
267 return nil, err
268 }
269 defer rows.Close()
270 out := map[string]Build{}
271 for rows.Next() {
272 b, err := scanBuild(rows)
273 if err != nil {
274 return nil, err
275 }
276 out[b.Job] = b // ascending: the last row for a job wins
277 }
278 return out, rows.Err()
279}
280
281// Elapsed reports how long a build ran. Zero until it has both a start and
282// a finish, which is every state but success and failure.
283func (b Build) Elapsed() time.Duration {
284 const layout = "2006-01-02T15:04:05Z"
285 start, err := time.Parse(layout, b.StartedAt)
286 if err != nil {
287 return 0
288 }
289 end, err := time.Parse(layout, b.FinishedAt)
290 if err != nil {
291 return 0
292 }
293 if d := end.Sub(start); d > 0 {
294 return d.Round(time.Second)
295 }
296 return 0
297}
298
299// CancelBuild withdraws a queued or running build. A running one is
300// ended by the runner, which learns of the cancellation when its log
301// session is closed, and whose later report lands on a row that already
302// says cancelled.
303func (s *Store) CancelBuild(id int64) error {
304 res, err := s.DB.Exec(`UPDATE builds SET status = 'cancelled',
305 finished_at = strftime('%Y-%m-%dT%H:%M:%SZ','now') WHERE id = ? AND status IN ('pending', 'running')`, id)
306 if err != nil {
307 return err
308 }
309 if n, _ := res.RowsAffected(); n == 0 {
310 return ErrNotFound
311 }
312 return nil
313}
314
315// SuccessBuildFor finds a passed build of the commit for the job, on any
316// ref: what a cancelled duplicate can point back at.
317func (s *Store) SuccessBuildFor(repoID int64, sha, job string) (Build, bool, error) {
318 b, err := scanBuild(s.DB.QueryRow(buildSelect+
319 " WHERE repo_id = ? AND sha = ? AND job = ? AND status = 'success' ORDER BY number DESC LIMIT 1", repoID, sha, job))
320 if errors.Is(err, sql.ErrNoRows) {
321 return Build{}, false, nil
322 }
323 return b, err == nil, err
324}