internal/store/builds.go
325 lines · 10318 bytes
1package store
2
3import (
4 "database/sql"
5 "errors"
6 "os"
7 "strconv"
8 "strings"
9 "time"
10)
11
12// Build is one CI job execution for one commit.
13type Build struct {
14 ID int64
15 RepoID int64
16 Number int64
17 Job string
18 SHA string
19 Ref string
20 Steps string // JSON array of shell commands
21 Image string // container image for the steps; "" means the runner default
22 Status string // pending|running|success|failure
23 CreatedAt string
24 StartedAt string
25 FinishedAt string
26 // Trusted is false for a merge request head fetched from another
27 // repository: its steps run without the target's secrets.
28 Trusted bool
29}
30
31// MaxBuildLog caps a build's stored log; appends past it are dropped.
32const MaxBuildLog = 2 << 20
33
34// truncNotice is appended once when a log first hits the cap. A log that
35// simply stops is indistinguishable from a build that died mid-step, which
36// is the reading that sent people hunting for a nonexistent test failure.
37var truncNotice = []byte("\n[log truncated: reached the " +
38 strconv.Itoa(MaxBuildLog>>20) + " MiB cap; earlier output is above]\n")
39
40// CreateBuild allocates the per-repo build number in the same transaction
41// as the insert, like issue and MR numbers.
42func (s *Store) CreateBuild(repoID int64, job, sha, ref, stepsJSON, image string, trusted bool) (int64, error) {
43 tx, err := s.DB.Begin()
44 if err != nil {
45 return 0, err
46 }
47 defer tx.Rollback()
48 if _, err := tx.Exec("UPDATE repos SET build_counter = build_counter + 1 WHERE id = ?", repoID); err != nil {
49 return 0, err
50 }
51 var n int64
52 if err := tx.QueryRow("SELECT build_counter FROM repos WHERE id = ?", repoID).Scan(&n); err != nil {
53 return 0, err
54 }
55 if _, err := tx.Exec(
56 "INSERT INTO builds (repo_id, number, job, sha, ref, steps, image, trusted) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
57 repoID, n, job, sha, ref, stepsJSON, image, trusted); err != nil {
58 return 0, err
59 }
60 return n, tx.Commit()
61}
62
63const buildSelect = `
64 SELECT id, repo_id, number, job, sha, ref, steps, image, status, created_at, started_at, finished_at, trusted
65 FROM builds`
66
67func scanBuild(row interface{ Scan(...any) error }) (Build, error) {
68 var b Build
69 var trusted int
70 err := row.Scan(&b.ID, &b.RepoID, &b.Number, &b.Job, &b.SHA, &b.Ref, &b.Steps, &b.Image,
71 &b.Status, &b.CreatedAt, &b.StartedAt, &b.FinishedAt, &trusted)
72 b.Trusted = trusted != 0
73 return b, err
74}
75
76// ClaimBuild atomically hands the oldest pending build to a runner.
77// ClaimBuild takes the oldest pending build and marks it running. A
78// non-empty repoIDs restricts the claim to those repositories, which is how
79// a runner on a machine that should not execute every repository's steps
80// limits what it picks up.
81func (s *Store) ClaimBuild(repoIDs []int64) (Build, bool, error) {
82 tx, err := s.DB.Begin()
83 if err != nil {
84 return Build{}, false, err
85 }
86 defer tx.Rollback()
87 query := "SELECT id FROM builds WHERE status = 'pending' ORDER BY id LIMIT 1"
88 args := []any{}
89 if len(repoIDs) > 0 {
90 marks := strings.TrimSuffix(strings.Repeat("?,", len(repoIDs)), ",")
91 query = "SELECT id FROM builds WHERE status = 'pending' AND repo_id IN (" +
92 marks + ") ORDER BY id LIMIT 1"
93 for _, id := range repoIDs {
94 args = append(args, id)
95 }
96 }
97 var id int64
98 err = tx.QueryRow(query, args...).Scan(&id)
99 if errors.Is(err, sql.ErrNoRows) {
100 return Build{}, false, nil
101 }
102 if err != nil {
103 return Build{}, false, err
104 }
105 if _, err := tx.Exec(
106 "UPDATE builds SET status = 'running', started_at = strftime('%Y-%m-%dT%H:%M:%SZ','now') WHERE id = ?", id); err != nil {
107 return Build{}, false, err
108 }
109 b, err := scanBuild(tx.QueryRow(buildSelect+" WHERE id = ?", id))
110 if err != nil {
111 return Build{}, false, err
112 }
113 return b, true, tx.Commit()
114}
115
116// StaleBuildDeadline is how long a claimed build may stay running before the
117// server gives up on it. Comfortably longer than the runner's own -timeout
118// (45m by default), so this only fires when the runner never reported at all —
119// it was killed, restarted, or lost the network mid-build.
120const StaleBuildDeadline = 90 * time.Minute
121
122// staleBuildDeadline is StaleBuildDeadline unless GITBAY_STALE_BUILD_DEADLINE
123// shortens it, which tests do.
124func staleBuildDeadline() time.Duration {
125 if v := os.Getenv("GITBAY_STALE_BUILD_DEADLINE"); v != "" {
126 if d, err := time.ParseDuration(v); err == nil && d > 0 {
127 return d
128 }
129 }
130 return StaleBuildDeadline
131}
132
133// ReapStaleBuilds fails every build that has been running past the deadline and
134// returns them, so the caller can resolve their commit statuses. A runner that
135// dies between claiming a build and reporting it otherwise leaves the row
136// claimed forever, and the commit pending forever with it.
137func (s *Store) ReapStaleBuilds() ([]Build, error) {
138 cutoff := time.Now().UTC().Add(-staleBuildDeadline()).Format("2006-01-02T15:04:05Z")
139 rows, err := s.DB.Query(buildSelect+
140 " WHERE status = 'running' AND started_at != '' AND started_at < ?", cutoff)
141 if err != nil {
142 return nil, err
143 }
144 defer rows.Close()
145 var stale []Build
146 for rows.Next() {
147 b, err := scanBuild(rows)
148 if err != nil {
149 return nil, err
150 }
151 stale = append(stale, b)
152 }
153 if err := rows.Err(); err != nil {
154 return nil, err
155 }
156 for _, b := range stale {
157 if err := s.AppendBuildLog(b.ID, []byte(
158 "\nbuild abandoned: the runner never reported an outcome\n")); err != nil {
159 return nil, err
160 }
161 if err := s.FinishBuild(b.ID, "failure"); err != nil {
162 return nil, err
163 }
164 }
165 return stale, nil
166}
167
168// AppendBuildLog adds a chunk to the build's log, dropping bytes past the cap.
169func (s *Store) AppendBuildLog(id int64, chunk []byte) error {
170 res, err := s.DB.Exec(`
171 UPDATE builds SET log = log || ?
172 WHERE id = ? AND length(log) < ?`, chunk, id, MaxBuildLog)
173 if err != nil {
174 return err
175 }
176 if n, _ := res.RowsAffected(); n > 0 {
177 return nil
178 }
179 // Over the cap. The bounds match exactly once: appending the notice puts
180 // the log past the upper bound, so later chunks fall through silently.
181 _, err = s.DB.Exec(`
182 UPDATE builds SET log = log || ?
183 WHERE id = ? AND length(log) >= ? AND length(log) < ?`,
184 truncNotice, id, MaxBuildLog, MaxBuildLog+len(truncNotice))
185 return err
186}
187
188// FinishBuild records the outcome of a running build.
189func (s *Store) FinishBuild(id int64, status string) error {
190 res, err := s.DB.Exec(`
191 UPDATE builds SET status = ?, finished_at = strftime('%Y-%m-%dT%H:%M:%SZ','now')
192 WHERE id = ? AND status = 'running'`, status, id)
193 if err != nil {
194 return err
195 }
196 if n, _ := res.RowsAffected(); n == 0 {
197 return ErrNotFound
198 }
199 return nil
200}
201
202func (s *Store) BuildByID(id int64) (Build, error) {
203 b, err := scanBuild(s.DB.QueryRow(buildSelect+" WHERE id = ?", id))
204 if errors.Is(err, sql.ErrNoRows) {
205 return b, ErrNotFound
206 }
207 return b, err
208}
209
210func (s *Store) BuildByNumber(repoID, number int64) (Build, error) {
211 b, err := scanBuild(s.DB.QueryRow(buildSelect+" WHERE repo_id = ? AND number = ?", repoID, number))
212 if errors.Is(err, sql.ErrNoRows) {
213 return b, ErrNotFound
214 }
215 return b, err
216}
217
218func (s *Store) ListBuilds(repoID int64, limit int) ([]Build, error) {
219 rows, err := s.DB.Query(buildSelect+" WHERE repo_id = ? ORDER BY number DESC LIMIT ?", repoID, limit)
220 if err != nil {
221 return nil, err
222 }
223 defer rows.Close()
224 var out []Build
225 for rows.Next() {
226 b, err := scanBuild(rows)
227 if err != nil {
228 return nil, err
229 }
230 out = append(out, b)
231 }
232 return out, rows.Err()
233}
234
235// BuildLog returns the stored log bytes.
236func (s *Store) BuildLog(id int64) ([]byte, error) {
237 var log []byte
238 err := s.DB.QueryRow("SELECT log FROM builds WHERE id = ?", id).Scan(&log)
239 if errors.Is(err, sql.ErrNoRows) {
240 return nil, ErrNotFound
241 }
242 return log, err
243}
244
245// LatestBuild returns the newest build for a repo, optionally narrowed to
246// one job. It is what a status badge reports.
247func (s *Store) LatestBuild(repoID int64, job string) (Build, error) {
248 q := buildSelect + " WHERE repo_id = ?"
249 args := []any{repoID}
250 if job != "" {
251 q += " AND job = ?"
252 args = append(args, job)
253 }
254 q += " ORDER BY number DESC LIMIT 1"
255 b, err := scanBuild(s.DB.QueryRow(q, args...))
256 if errors.Is(err, sql.ErrNoRows) {
257 return b, ErrNotFound
258 }
259 return b, err
260}
261
262// BuildsForCommit returns the newest build per job for one commit. A merge
263// request's checks are ci/<job> statuses; this is where their timing comes
264// from, in one query rather than one per check.
265func (s *Store) BuildsForCommit(repoID int64, sha string) (map[string]Build, error) {
266 rows, err := s.DB.Query(buildSelect+" WHERE repo_id = ? AND sha = ? ORDER BY number ASC", repoID, sha)
267 if err != nil {
268 return nil, err
269 }
270 defer rows.Close()
271 out := map[string]Build{}
272 for rows.Next() {
273 b, err := scanBuild(rows)
274 if err != nil {
275 return nil, err
276 }
277 out[b.Job] = b // ascending: the last row for a job wins
278 }
279 return out, rows.Err()
280}
281
282// Elapsed reports how long a build ran. Zero until it has both a start and
283// a finish, which is every state but success and failure.
284func (b Build) Elapsed() time.Duration {
285 const layout = "2006-01-02T15:04:05Z"
286 start, err := time.Parse(layout, b.StartedAt)
287 if err != nil {
288 return 0
289 }
290 end, err := time.Parse(layout, b.FinishedAt)
291 if err != nil {
292 return 0
293 }
294 if d := end.Sub(start); d > 0 {
295 return d.Round(time.Second)
296 }
297 return 0
298}
299
300// CancelBuild withdraws a queued or running build. A running one is
301// ended by the runner, which learns of the cancellation when its log
302// session is closed, and whose later report lands on a row that already
303// says cancelled.
304func (s *Store) CancelBuild(id int64) error {
305 res, err := s.DB.Exec(`UPDATE builds SET status = 'cancelled',
306 finished_at = strftime('%Y-%m-%dT%H:%M:%SZ','now') WHERE id = ? AND status IN ('pending', 'running')`, id)
307 if err != nil {
308 return err
309 }
310 if n, _ := res.RowsAffected(); n == 0 {
311 return ErrNotFound
312 }
313 return nil
314}
315
316// SuccessBuildFor finds a passed build of the commit for the job, on any
317// ref: what a cancelled duplicate can point back at.
318func (s *Store) SuccessBuildFor(repoID int64, sha, job string) (Build, bool, error) {
319 b, err := scanBuild(s.DB.QueryRow(buildSelect+
320 " WHERE repo_id = ? AND sha = ? AND job = ? AND status = 'success' ORDER BY number DESC LIMIT 1", repoID, sha, job))
321 if errors.Is(err, sql.ErrNoRows) {
322 return Build{}, false, nil
323 }
324 return b, err == nil, err
325}