internal/store/builds.go
341 lines · 11019 bytes
1package store
2
3import (
4 "database/sql"
5 "errors"
6 "os"
7 "strconv"
8 "strings"
9 "time"
10)
11
12// Build is one CI job execution for one commit.
13type Build struct {
14 ID int64
15 RepoID int64
16 Number int64
17 Job string
18 SHA string
19 Ref string
20 Steps string // JSON array of shell commands
21 Image string // container image for the steps; "" means the runner default
22 Tree string // the commit's tree; "" when not deduplicated by tree
23 Status string // pending|running|success|failure
24 CreatedAt string
25 StartedAt string
26 FinishedAt string
27 // Trusted is false for a merge request head fetched from another
28 // repository: its steps run without the target's secrets.
29 Trusted bool
30}
31
32// MaxBuildLog caps a build's stored log; appends past it are dropped.
33const MaxBuildLog = 2 << 20
34
35// truncNotice is appended once when a log first hits the cap. A log that
36// simply stops is indistinguishable from a build that died mid-step, which
37// is the reading that sent people hunting for a nonexistent test failure.
38var truncNotice = []byte("\n[log truncated: reached the " +
39 strconv.Itoa(MaxBuildLog>>20) + " MiB cap; earlier output is above]\n")
40
41// CreateBuild allocates the per-repo build number in the same transaction
42// as the insert, like issue and MR numbers.
43func (s *Store) CreateBuild(repoID int64, job, sha, ref, stepsJSON, image, tree string, trusted bool) (int64, error) {
44 tx, err := s.DB.Begin()
45 if err != nil {
46 return 0, err
47 }
48 defer tx.Rollback()
49 if _, err := tx.Exec("UPDATE repos SET build_counter = build_counter + 1 WHERE id = ?", repoID); err != nil {
50 return 0, err
51 }
52 var n int64
53 if err := tx.QueryRow("SELECT build_counter FROM repos WHERE id = ?", repoID).Scan(&n); err != nil {
54 return 0, err
55 }
56 if _, err := tx.Exec(
57 "INSERT INTO builds (repo_id, number, job, sha, ref, steps, image, tree, trusted) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
58 repoID, n, job, sha, ref, stepsJSON, image, tree, trusted); err != nil {
59 return 0, err
60 }
61 return n, tx.Commit()
62}
63
64const buildSelect = `
65 SELECT id, repo_id, number, job, sha, ref, steps, image, tree, status, created_at, started_at, finished_at, trusted
66 FROM builds`
67
68func scanBuild(row interface{ Scan(...any) error }) (Build, error) {
69 var b Build
70 var trusted int
71 err := row.Scan(&b.ID, &b.RepoID, &b.Number, &b.Job, &b.SHA, &b.Ref, &b.Steps, &b.Image, &b.Tree,
72 &b.Status, &b.CreatedAt, &b.StartedAt, &b.FinishedAt, &trusted)
73 b.Trusted = trusted != 0
74 return b, err
75}
76
77// ClaimBuild atomically hands the oldest pending build to a runner.
78// ClaimBuild takes the oldest pending build and marks it running. A
79// non-empty repoIDs restricts the claim to those repositories, which is how
80// a runner on a machine that should not execute every repository's steps
81// limits what it picks up.
82func (s *Store) ClaimBuild(repoIDs []int64) (Build, bool, error) {
83 tx, err := s.DB.Begin()
84 if err != nil {
85 return Build{}, false, err
86 }
87 defer tx.Rollback()
88 query := "SELECT id FROM builds WHERE status = 'pending' ORDER BY id LIMIT 1"
89 args := []any{}
90 if len(repoIDs) > 0 {
91 marks := strings.TrimSuffix(strings.Repeat("?,", len(repoIDs)), ",")
92 query = "SELECT id FROM builds WHERE status = 'pending' AND repo_id IN (" +
93 marks + ") ORDER BY id LIMIT 1"
94 for _, id := range repoIDs {
95 args = append(args, id)
96 }
97 }
98 var id int64
99 err = tx.QueryRow(query, args...).Scan(&id)
100 if errors.Is(err, sql.ErrNoRows) {
101 return Build{}, false, nil
102 }
103 if err != nil {
104 return Build{}, false, err
105 }
106 if _, err := tx.Exec(
107 "UPDATE builds SET status = 'running', started_at = strftime('%Y-%m-%dT%H:%M:%SZ','now') WHERE id = ?", id); err != nil {
108 return Build{}, false, err
109 }
110 b, err := scanBuild(tx.QueryRow(buildSelect+" WHERE id = ?", id))
111 if err != nil {
112 return Build{}, false, err
113 }
114 return b, true, tx.Commit()
115}
116
117// StaleBuildDeadline is how long a claimed build may stay running before the
118// server gives up on it. Comfortably longer than the runner's own -timeout
119// (45m by default), so this only fires when the runner never reported at all —
120// it was killed, restarted, or lost the network mid-build.
121const StaleBuildDeadline = 90 * time.Minute
122
123// staleBuildDeadline is StaleBuildDeadline unless GITBAY_STALE_BUILD_DEADLINE
124// shortens it, which tests do.
125func staleBuildDeadline() time.Duration {
126 if v := os.Getenv("GITBAY_STALE_BUILD_DEADLINE"); v != "" {
127 if d, err := time.ParseDuration(v); err == nil && d > 0 {
128 return d
129 }
130 }
131 return StaleBuildDeadline
132}
133
134// ReapStaleBuilds fails every build that has been running past the deadline and
135// returns them, so the caller can resolve their commit statuses. A runner that
136// dies between claiming a build and reporting it otherwise leaves the row
137// claimed forever, and the commit pending forever with it.
138func (s *Store) ReapStaleBuilds() ([]Build, error) {
139 cutoff := time.Now().UTC().Add(-staleBuildDeadline()).Format("2006-01-02T15:04:05Z")
140 rows, err := s.DB.Query(buildSelect+
141 " WHERE status = 'running' AND started_at != '' AND started_at < ?", cutoff)
142 if err != nil {
143 return nil, err
144 }
145 defer rows.Close()
146 var stale []Build
147 for rows.Next() {
148 b, err := scanBuild(rows)
149 if err != nil {
150 return nil, err
151 }
152 stale = append(stale, b)
153 }
154 if err := rows.Err(); err != nil {
155 return nil, err
156 }
157 for _, b := range stale {
158 if err := s.AppendBuildLog(b.ID, []byte(
159 "\nbuild abandoned: the runner never reported an outcome\n")); err != nil {
160 return nil, err
161 }
162 if err := s.FinishBuild(b.ID, "failure"); err != nil {
163 return nil, err
164 }
165 }
166 return stale, nil
167}
168
169// AppendBuildLog adds a chunk to the build's log, dropping bytes past the cap.
170func (s *Store) AppendBuildLog(id int64, chunk []byte) error {
171 res, err := s.DB.Exec(`
172 UPDATE builds SET log = log || ?
173 WHERE id = ? AND length(log) < ?`, chunk, id, MaxBuildLog)
174 if err != nil {
175 return err
176 }
177 if n, _ := res.RowsAffected(); n > 0 {
178 return nil
179 }
180 // Over the cap. The bounds match exactly once: appending the notice puts
181 // the log past the upper bound, so later chunks fall through silently.
182 _, err = s.DB.Exec(`
183 UPDATE builds SET log = log || ?
184 WHERE id = ? AND length(log) >= ? AND length(log) < ?`,
185 truncNotice, id, MaxBuildLog, MaxBuildLog+len(truncNotice))
186 return err
187}
188
189// FinishBuild records the outcome of a running build.
190func (s *Store) FinishBuild(id int64, status string) error {
191 res, err := s.DB.Exec(`
192 UPDATE builds SET status = ?, finished_at = strftime('%Y-%m-%dT%H:%M:%SZ','now')
193 WHERE id = ? AND status = 'running'`, status, id)
194 if err != nil {
195 return err
196 }
197 if n, _ := res.RowsAffected(); n == 0 {
198 return ErrNotFound
199 }
200 return nil
201}
202
203func (s *Store) BuildByID(id int64) (Build, error) {
204 b, err := scanBuild(s.DB.QueryRow(buildSelect+" WHERE id = ?", id))
205 if errors.Is(err, sql.ErrNoRows) {
206 return b, ErrNotFound
207 }
208 return b, err
209}
210
211func (s *Store) BuildByNumber(repoID, number int64) (Build, error) {
212 b, err := scanBuild(s.DB.QueryRow(buildSelect+" WHERE repo_id = ? AND number = ?", repoID, number))
213 if errors.Is(err, sql.ErrNoRows) {
214 return b, ErrNotFound
215 }
216 return b, err
217}
218
219func (s *Store) ListBuilds(repoID int64, limit int) ([]Build, error) {
220 rows, err := s.DB.Query(buildSelect+" WHERE repo_id = ? ORDER BY number DESC LIMIT ?", repoID, limit)
221 if err != nil {
222 return nil, err
223 }
224 defer rows.Close()
225 var out []Build
226 for rows.Next() {
227 b, err := scanBuild(rows)
228 if err != nil {
229 return nil, err
230 }
231 out = append(out, b)
232 }
233 return out, rows.Err()
234}
235
236// BuildLog returns the stored log bytes.
237func (s *Store) BuildLog(id int64) ([]byte, error) {
238 var log []byte
239 err := s.DB.QueryRow("SELECT log FROM builds WHERE id = ?", id).Scan(&log)
240 if errors.Is(err, sql.ErrNoRows) {
241 return nil, ErrNotFound
242 }
243 return log, err
244}
245
246// LatestBuild returns the newest build for a repo, optionally narrowed to
247// one job. It is what a status badge reports.
248func (s *Store) LatestBuild(repoID int64, job string) (Build, error) {
249 q := buildSelect + " WHERE repo_id = ?"
250 args := []any{repoID}
251 if job != "" {
252 q += " AND job = ?"
253 args = append(args, job)
254 }
255 q += " ORDER BY number DESC LIMIT 1"
256 b, err := scanBuild(s.DB.QueryRow(q, args...))
257 if errors.Is(err, sql.ErrNoRows) {
258 return b, ErrNotFound
259 }
260 return b, err
261}
262
263// BuildsForCommit returns the newest build per job for one commit. A merge
264// request's checks are ci/<job> statuses; this is where their timing comes
265// from, in one query rather than one per check.
266func (s *Store) BuildsForCommit(repoID int64, sha string) (map[string]Build, error) {
267 rows, err := s.DB.Query(buildSelect+" WHERE repo_id = ? AND sha = ? ORDER BY number ASC", repoID, sha)
268 if err != nil {
269 return nil, err
270 }
271 defer rows.Close()
272 out := map[string]Build{}
273 for rows.Next() {
274 b, err := scanBuild(rows)
275 if err != nil {
276 return nil, err
277 }
278 out[b.Job] = b // ascending: the last row for a job wins
279 }
280 return out, rows.Err()
281}
282
283// Elapsed reports how long a build ran. Zero until it has both a start and
284// a finish, which is every state but success and failure.
285func (b Build) Elapsed() time.Duration {
286 const layout = "2006-01-02T15:04:05Z"
287 start, err := time.Parse(layout, b.StartedAt)
288 if err != nil {
289 return 0
290 }
291 end, err := time.Parse(layout, b.FinishedAt)
292 if err != nil {
293 return 0
294 }
295 if d := end.Sub(start); d > 0 {
296 return d.Round(time.Second)
297 }
298 return 0
299}
300
301// CancelBuild withdraws a queued or running build. A running one is
302// ended by the runner, which learns of the cancellation when its log
303// session is closed, and whose later report lands on a row that already
304// says cancelled.
305func (s *Store) CancelBuild(id int64) error {
306 res, err := s.DB.Exec(`UPDATE builds SET status = 'cancelled',
307 finished_at = strftime('%Y-%m-%dT%H:%M:%SZ','now') WHERE id = ? AND status IN ('pending', 'running')`, id)
308 if err != nil {
309 return err
310 }
311 if n, _ := res.RowsAffected(); n == 0 {
312 return ErrNotFound
313 }
314 return nil
315}
316
317// SuccessBuildFor finds a passed build of the commit for the job, on any
318// ref: what a cancelled duplicate can point back at.
319// SuccessBuildForTree is SuccessBuildFor keyed by tree rather than
320// commit: a rebase that changes nothing in the tree has already been
321// built (#177). An empty tree never matches.
322func (s *Store) SuccessBuildForTree(repoID int64, tree, job string) (Build, bool, error) {
323 if tree == "" {
324 return Build{}, false, nil
325 }
326 b, err := scanBuild(s.DB.QueryRow(buildSelect+
327 " WHERE repo_id = ? AND tree = ? AND job = ? AND status = 'success' ORDER BY number DESC LIMIT 1", repoID, tree, job))
328 if errors.Is(err, sql.ErrNoRows) {
329 return Build{}, false, nil
330 }
331 return b, err == nil, err
332}
333
334func (s *Store) SuccessBuildFor(repoID int64, sha, job string) (Build, bool, error) {
335 b, err := scanBuild(s.DB.QueryRow(buildSelect+
336 " WHERE repo_id = ? AND sha = ? AND job = ? AND status = 'success' ORDER BY number DESC LIMIT 1", repoID, sha, job))
337 if errors.Is(err, sql.ErrNoRows) {
338 return Build{}, false, nil
339 }
340 return b, err == nil, err
341}