internal/store/builds.go
319 lines · 9958 bytes
1package store
2
3import (
4 "database/sql"
5 "errors"
6 "os"
7 "strconv"
8 "strings"
9 "time"
10)
11
12// Build is one CI job execution for one commit.
13type Build struct {
14 ID int64
15 RepoID int64
16 Number int64
17 Job string
18 SHA string
19 Ref string
20 Steps string // JSON array of shell commands
21 Status string // pending|running|success|failure
22 CreatedAt string
23 StartedAt string
24 FinishedAt string
25}
26
27// MaxBuildLog caps a build's stored log; appends past it are dropped.
28const MaxBuildLog = 2 << 20
29
30// truncNotice is appended once when a log first hits the cap. A log that
31// simply stops is indistinguishable from a build that died mid-step, which
32// is the reading that sent people hunting for a nonexistent test failure.
33var truncNotice = []byte("\n[log truncated: reached the " +
34 strconv.Itoa(MaxBuildLog>>20) + " MiB cap; earlier output is above]\n")
35
36// CreateBuild allocates the per-repo build number in the same transaction
37// as the insert, like issue and MR numbers.
38func (s *Store) CreateBuild(repoID int64, job, sha, ref, stepsJSON string) (int64, error) {
39 tx, err := s.DB.Begin()
40 if err != nil {
41 return 0, err
42 }
43 defer tx.Rollback()
44 if _, err := tx.Exec("UPDATE repos SET build_counter = build_counter + 1 WHERE id = ?", repoID); err != nil {
45 return 0, err
46 }
47 var n int64
48 if err := tx.QueryRow("SELECT build_counter FROM repos WHERE id = ?", repoID).Scan(&n); err != nil {
49 return 0, err
50 }
51 if _, err := tx.Exec(
52 "INSERT INTO builds (repo_id, number, job, sha, ref, steps) VALUES (?, ?, ?, ?, ?, ?)",
53 repoID, n, job, sha, ref, stepsJSON); err != nil {
54 return 0, err
55 }
56 return n, tx.Commit()
57}
58
59const buildSelect = `
60 SELECT id, repo_id, number, job, sha, ref, steps, status, created_at, started_at, finished_at
61 FROM builds`
62
63func scanBuild(row interface{ Scan(...any) error }) (Build, error) {
64 var b Build
65 err := row.Scan(&b.ID, &b.RepoID, &b.Number, &b.Job, &b.SHA, &b.Ref, &b.Steps,
66 &b.Status, &b.CreatedAt, &b.StartedAt, &b.FinishedAt)
67 return b, err
68}
69
70// ClaimBuild atomically hands the oldest pending build to a runner.
71// ClaimBuild takes the oldest pending build and marks it running. A
72// non-empty repoIDs restricts the claim to those repositories, which is how
73// a runner on a machine that should not execute every repository's steps
74// limits what it picks up.
75func (s *Store) ClaimBuild(repoIDs []int64) (Build, bool, error) {
76 tx, err := s.DB.Begin()
77 if err != nil {
78 return Build{}, false, err
79 }
80 defer tx.Rollback()
81 query := "SELECT id FROM builds WHERE status = 'pending' ORDER BY id LIMIT 1"
82 args := []any{}
83 if len(repoIDs) > 0 {
84 marks := strings.TrimSuffix(strings.Repeat("?,", len(repoIDs)), ",")
85 query = "SELECT id FROM builds WHERE status = 'pending' AND repo_id IN (" +
86 marks + ") ORDER BY id LIMIT 1"
87 for _, id := range repoIDs {
88 args = append(args, id)
89 }
90 }
91 var id int64
92 err = tx.QueryRow(query, args...).Scan(&id)
93 if errors.Is(err, sql.ErrNoRows) {
94 return Build{}, false, nil
95 }
96 if err != nil {
97 return Build{}, false, err
98 }
99 if _, err := tx.Exec(
100 "UPDATE builds SET status = 'running', started_at = strftime('%Y-%m-%dT%H:%M:%SZ','now') WHERE id = ?", id); err != nil {
101 return Build{}, false, err
102 }
103 b, err := scanBuild(tx.QueryRow(buildSelect+" WHERE id = ?", id))
104 if err != nil {
105 return Build{}, false, err
106 }
107 return b, true, tx.Commit()
108}
109
110// StaleBuildDeadline is how long a claimed build may stay running before the
111// server gives up on it. Comfortably longer than the runner's own -timeout
112// (45m by default), so this only fires when the runner never reported at all —
113// it was killed, restarted, or lost the network mid-build.
114const StaleBuildDeadline = 90 * time.Minute
115
116// staleBuildDeadline is StaleBuildDeadline unless GITBAY_STALE_BUILD_DEADLINE
117// shortens it, which tests do.
118func staleBuildDeadline() time.Duration {
119 if v := os.Getenv("GITBAY_STALE_BUILD_DEADLINE"); v != "" {
120 if d, err := time.ParseDuration(v); err == nil && d > 0 {
121 return d
122 }
123 }
124 return StaleBuildDeadline
125}
126
127// ReapStaleBuilds fails every build that has been running past the deadline and
128// returns them, so the caller can resolve their commit statuses. A runner that
129// dies between claiming a build and reporting it otherwise leaves the row
130// claimed forever, and the commit pending forever with it.
131func (s *Store) ReapStaleBuilds() ([]Build, error) {
132 cutoff := time.Now().UTC().Add(-staleBuildDeadline()).Format("2006-01-02T15:04:05Z")
133 rows, err := s.DB.Query(buildSelect+
134 " WHERE status = 'running' AND started_at != '' AND started_at < ?", cutoff)
135 if err != nil {
136 return nil, err
137 }
138 defer rows.Close()
139 var stale []Build
140 for rows.Next() {
141 b, err := scanBuild(rows)
142 if err != nil {
143 return nil, err
144 }
145 stale = append(stale, b)
146 }
147 if err := rows.Err(); err != nil {
148 return nil, err
149 }
150 for _, b := range stale {
151 if err := s.AppendBuildLog(b.ID, []byte(
152 "\nbuild abandoned: the runner never reported an outcome\n")); err != nil {
153 return nil, err
154 }
155 if err := s.FinishBuild(b.ID, "failure"); err != nil {
156 return nil, err
157 }
158 }
159 return stale, nil
160}
161
162// AppendBuildLog adds a chunk to the build's log, dropping bytes past the cap.
163func (s *Store) AppendBuildLog(id int64, chunk []byte) error {
164 res, err := s.DB.Exec(`
165 UPDATE builds SET log = log || ?
166 WHERE id = ? AND length(log) < ?`, chunk, id, MaxBuildLog)
167 if err != nil {
168 return err
169 }
170 if n, _ := res.RowsAffected(); n > 0 {
171 return nil
172 }
173 // Over the cap. The bounds match exactly once: appending the notice puts
174 // the log past the upper bound, so later chunks fall through silently.
175 _, err = s.DB.Exec(`
176 UPDATE builds SET log = log || ?
177 WHERE id = ? AND length(log) >= ? AND length(log) < ?`,
178 truncNotice, id, MaxBuildLog, MaxBuildLog+len(truncNotice))
179 return err
180}
181
182// FinishBuild records the outcome of a running build.
183func (s *Store) FinishBuild(id int64, status string) error {
184 res, err := s.DB.Exec(`
185 UPDATE builds SET status = ?, finished_at = strftime('%Y-%m-%dT%H:%M:%SZ','now')
186 WHERE id = ? AND status = 'running'`, status, id)
187 if err != nil {
188 return err
189 }
190 if n, _ := res.RowsAffected(); n == 0 {
191 return ErrNotFound
192 }
193 return nil
194}
195
196func (s *Store) BuildByID(id int64) (Build, error) {
197 b, err := scanBuild(s.DB.QueryRow(buildSelect+" WHERE id = ?", id))
198 if errors.Is(err, sql.ErrNoRows) {
199 return b, ErrNotFound
200 }
201 return b, err
202}
203
204func (s *Store) BuildByNumber(repoID, number int64) (Build, error) {
205 b, err := scanBuild(s.DB.QueryRow(buildSelect+" WHERE repo_id = ? AND number = ?", repoID, number))
206 if errors.Is(err, sql.ErrNoRows) {
207 return b, ErrNotFound
208 }
209 return b, err
210}
211
212func (s *Store) ListBuilds(repoID int64, limit int) ([]Build, error) {
213 rows, err := s.DB.Query(buildSelect+" WHERE repo_id = ? ORDER BY number DESC LIMIT ?", repoID, limit)
214 if err != nil {
215 return nil, err
216 }
217 defer rows.Close()
218 var out []Build
219 for rows.Next() {
220 b, err := scanBuild(rows)
221 if err != nil {
222 return nil, err
223 }
224 out = append(out, b)
225 }
226 return out, rows.Err()
227}
228
229// BuildLog returns the stored log bytes.
230func (s *Store) BuildLog(id int64) ([]byte, error) {
231 var log []byte
232 err := s.DB.QueryRow("SELECT log FROM builds WHERE id = ?", id).Scan(&log)
233 if errors.Is(err, sql.ErrNoRows) {
234 return nil, ErrNotFound
235 }
236 return log, err
237}
238
239// LatestBuild returns the newest build for a repo, optionally narrowed to
240// one job. It is what a status badge reports.
241func (s *Store) LatestBuild(repoID int64, job string) (Build, error) {
242 q := buildSelect + " WHERE repo_id = ?"
243 args := []any{repoID}
244 if job != "" {
245 q += " AND job = ?"
246 args = append(args, job)
247 }
248 q += " ORDER BY number DESC LIMIT 1"
249 b, err := scanBuild(s.DB.QueryRow(q, args...))
250 if errors.Is(err, sql.ErrNoRows) {
251 return b, ErrNotFound
252 }
253 return b, err
254}
255
256// BuildsForCommit returns the newest build per job for one commit. A merge
257// request's checks are ci/<job> statuses; this is where their timing comes
258// from, in one query rather than one per check.
259func (s *Store) BuildsForCommit(repoID int64, sha string) (map[string]Build, error) {
260 rows, err := s.DB.Query(buildSelect+" WHERE repo_id = ? AND sha = ? ORDER BY number ASC", repoID, sha)
261 if err != nil {
262 return nil, err
263 }
264 defer rows.Close()
265 out := map[string]Build{}
266 for rows.Next() {
267 b, err := scanBuild(rows)
268 if err != nil {
269 return nil, err
270 }
271 out[b.Job] = b // ascending: the last row for a job wins
272 }
273 return out, rows.Err()
274}
275
276// Elapsed reports how long a build ran. Zero until it has both a start and
277// a finish, which is every state but success and failure.
278func (b Build) Elapsed() time.Duration {
279 const layout = "2006-01-02T15:04:05Z"
280 start, err := time.Parse(layout, b.StartedAt)
281 if err != nil {
282 return 0
283 }
284 end, err := time.Parse(layout, b.FinishedAt)
285 if err != nil {
286 return 0
287 }
288 if d := end.Sub(start); d > 0 {
289 return d.Round(time.Second)
290 }
291 return 0
292}
293
294// CancelBuild withdraws a queued or running build. A running one is
295// ended by the runner, which learns of the cancellation when its log
296// session is closed, and whose later report lands on a row that already
297// says cancelled.
298func (s *Store) CancelBuild(id int64) error {
299 res, err := s.DB.Exec(`UPDATE builds SET status = 'cancelled',
300 finished_at = strftime('%Y-%m-%dT%H:%M:%SZ','now') WHERE id = ? AND status IN ('pending', 'running')`, id)
301 if err != nil {
302 return err
303 }
304 if n, _ := res.RowsAffected(); n == 0 {
305 return ErrNotFound
306 }
307 return nil
308}
309
310// SuccessBuildFor finds a passed build of the commit for the job, on any
311// ref: what a cancelled duplicate can point back at.
312func (s *Store) SuccessBuildFor(repoID int64, sha, job string) (Build, bool, error) {
313 b, err := scanBuild(s.DB.QueryRow(buildSelect+
314 " WHERE repo_id = ? AND sha = ? AND job = ? AND status = 'success' ORDER BY number DESC LIMIT 1", repoID, sha, job))
315 if errors.Is(err, sql.ErrNoRows) {
316 return Build{}, false, nil
317 }
318 return b, err == nil, err
319}