internal/store/builds.go
280 lines · 8585 bytes
1package store
2
3import (
4 "database/sql"
5 "errors"
6 "strconv"
7 "strings"
8 "time"
9)
10
11// Build is one CI job execution for one commit.
12type Build struct {
13 ID int64
14 RepoID int64
15 Number int64
16 Job string
17 SHA string
18 Ref string
19 Steps string // JSON array of shell commands
20 Status string // pending|running|success|failure
21 CreatedAt string
22 StartedAt string
23 FinishedAt string
24}
25
26// MaxBuildLog caps a build's stored log; appends past it are dropped.
27const MaxBuildLog = 2 << 20
28
29// truncNotice is appended once when a log first hits the cap. A log that
30// simply stops is indistinguishable from a build that died mid-step, which
31// is the reading that sent people hunting for a nonexistent test failure.
32var truncNotice = []byte("\n[log truncated: reached the " +
33 strconv.Itoa(MaxBuildLog>>20) + " MiB cap; earlier output is above]\n")
34
35// CreateBuild allocates the per-repo build number in the same transaction
36// as the insert, like issue and MR numbers.
37func (s *Store) CreateBuild(repoID int64, job, sha, ref, stepsJSON string) (int64, error) {
38 tx, err := s.DB.Begin()
39 if err != nil {
40 return 0, err
41 }
42 defer tx.Rollback()
43 if _, err := tx.Exec("UPDATE repos SET build_counter = build_counter + 1 WHERE id = ?", repoID); err != nil {
44 return 0, err
45 }
46 var n int64
47 if err := tx.QueryRow("SELECT build_counter FROM repos WHERE id = ?", repoID).Scan(&n); err != nil {
48 return 0, err
49 }
50 if _, err := tx.Exec(
51 "INSERT INTO builds (repo_id, number, job, sha, ref, steps) VALUES (?, ?, ?, ?, ?, ?)",
52 repoID, n, job, sha, ref, stepsJSON); err != nil {
53 return 0, err
54 }
55 return n, tx.Commit()
56}
57
58const buildSelect = `
59 SELECT id, repo_id, number, job, sha, ref, steps, status, created_at, started_at, finished_at
60 FROM builds`
61
62func scanBuild(row interface{ Scan(...any) error }) (Build, error) {
63 var b Build
64 err := row.Scan(&b.ID, &b.RepoID, &b.Number, &b.Job, &b.SHA, &b.Ref, &b.Steps,
65 &b.Status, &b.CreatedAt, &b.StartedAt, &b.FinishedAt)
66 return b, err
67}
68
69// ClaimBuild atomically hands the oldest pending build to a runner.
70// ClaimBuild takes the oldest pending build and marks it running. A
71// non-empty repoIDs restricts the claim to those repositories, which is how
72// a runner on a machine that should not execute every repository's steps
73// limits what it picks up.
74func (s *Store) ClaimBuild(repoIDs []int64) (Build, bool, error) {
75 tx, err := s.DB.Begin()
76 if err != nil {
77 return Build{}, false, err
78 }
79 defer tx.Rollback()
80 query := "SELECT id FROM builds WHERE status = 'pending' ORDER BY id LIMIT 1"
81 args := []any{}
82 if len(repoIDs) > 0 {
83 marks := strings.TrimSuffix(strings.Repeat("?,", len(repoIDs)), ",")
84 query = "SELECT id FROM builds WHERE status = 'pending' AND repo_id IN (" +
85 marks + ") ORDER BY id LIMIT 1"
86 for _, id := range repoIDs {
87 args = append(args, id)
88 }
89 }
90 var id int64
91 err = tx.QueryRow(query, args...).Scan(&id)
92 if errors.Is(err, sql.ErrNoRows) {
93 return Build{}, false, nil
94 }
95 if err != nil {
96 return Build{}, false, err
97 }
98 if _, err := tx.Exec(
99 "UPDATE builds SET status = 'running', started_at = strftime('%Y-%m-%dT%H:%M:%SZ','now') WHERE id = ?", id); err != nil {
100 return Build{}, false, err
101 }
102 b, err := scanBuild(tx.QueryRow(buildSelect+" WHERE id = ?", id))
103 if err != nil {
104 return Build{}, false, err
105 }
106 return b, true, tx.Commit()
107}
108
109// StaleBuildDeadline is how long a claimed build may stay running before the
110// server gives up on it. Comfortably longer than the runner's own -timeout
111// (45m by default), so this only fires when the runner never reported at all —
112// it was killed, restarted, or lost the network mid-build.
113const StaleBuildDeadline = 90 * time.Minute
114
115// ReapStaleBuilds fails every build that has been running past the deadline and
116// returns them, so the caller can resolve their commit statuses. A runner that
117// dies between claiming a build and reporting it otherwise leaves the row
118// claimed forever, and the commit pending forever with it.
119func (s *Store) ReapStaleBuilds() ([]Build, error) {
120 cutoff := time.Now().UTC().Add(-StaleBuildDeadline).Format("2006-01-02T15:04:05Z")
121 rows, err := s.DB.Query(buildSelect+
122 " WHERE status = 'running' AND started_at != '' AND started_at < ?", cutoff)
123 if err != nil {
124 return nil, err
125 }
126 defer rows.Close()
127 var stale []Build
128 for rows.Next() {
129 b, err := scanBuild(rows)
130 if err != nil {
131 return nil, err
132 }
133 stale = append(stale, b)
134 }
135 if err := rows.Err(); err != nil {
136 return nil, err
137 }
138 for _, b := range stale {
139 if err := s.AppendBuildLog(b.ID, []byte(
140 "\nbuild abandoned: the runner never reported an outcome\n")); err != nil {
141 return nil, err
142 }
143 if err := s.FinishBuild(b.ID, "failure"); err != nil {
144 return nil, err
145 }
146 }
147 return stale, nil
148}
149
150// AppendBuildLog adds a chunk to the build's log, dropping bytes past the cap.
151func (s *Store) AppendBuildLog(id int64, chunk []byte) error {
152 res, err := s.DB.Exec(`
153 UPDATE builds SET log = log || ?
154 WHERE id = ? AND length(log) < ?`, chunk, id, MaxBuildLog)
155 if err != nil {
156 return err
157 }
158 if n, _ := res.RowsAffected(); n > 0 {
159 return nil
160 }
161 // Over the cap. The bounds match exactly once: appending the notice puts
162 // the log past the upper bound, so later chunks fall through silently.
163 _, err = s.DB.Exec(`
164 UPDATE builds SET log = log || ?
165 WHERE id = ? AND length(log) >= ? AND length(log) < ?`,
166 truncNotice, id, MaxBuildLog, MaxBuildLog+len(truncNotice))
167 return err
168}
169
170// FinishBuild records the outcome of a running build.
171func (s *Store) FinishBuild(id int64, status string) error {
172 res, err := s.DB.Exec(`
173 UPDATE builds SET status = ?, finished_at = strftime('%Y-%m-%dT%H:%M:%SZ','now')
174 WHERE id = ? AND status = 'running'`, status, id)
175 if err != nil {
176 return err
177 }
178 if n, _ := res.RowsAffected(); n == 0 {
179 return ErrNotFound
180 }
181 return nil
182}
183
184func (s *Store) BuildByID(id int64) (Build, error) {
185 b, err := scanBuild(s.DB.QueryRow(buildSelect+" WHERE id = ?", id))
186 if errors.Is(err, sql.ErrNoRows) {
187 return b, ErrNotFound
188 }
189 return b, err
190}
191
192func (s *Store) BuildByNumber(repoID, number int64) (Build, error) {
193 b, err := scanBuild(s.DB.QueryRow(buildSelect+" WHERE repo_id = ? AND number = ?", repoID, number))
194 if errors.Is(err, sql.ErrNoRows) {
195 return b, ErrNotFound
196 }
197 return b, err
198}
199
200func (s *Store) ListBuilds(repoID int64, limit int) ([]Build, error) {
201 rows, err := s.DB.Query(buildSelect+" WHERE repo_id = ? ORDER BY number DESC LIMIT ?", repoID, limit)
202 if err != nil {
203 return nil, err
204 }
205 defer rows.Close()
206 var out []Build
207 for rows.Next() {
208 b, err := scanBuild(rows)
209 if err != nil {
210 return nil, err
211 }
212 out = append(out, b)
213 }
214 return out, rows.Err()
215}
216
217// BuildLog returns the stored log bytes.
218func (s *Store) BuildLog(id int64) ([]byte, error) {
219 var log []byte
220 err := s.DB.QueryRow("SELECT log FROM builds WHERE id = ?", id).Scan(&log)
221 if errors.Is(err, sql.ErrNoRows) {
222 return nil, ErrNotFound
223 }
224 return log, err
225}
226
227// LatestBuild returns the newest build for a repo, optionally narrowed to
228// one job. It is what a status badge reports.
229func (s *Store) LatestBuild(repoID int64, job string) (Build, error) {
230 q := buildSelect + " WHERE repo_id = ?"
231 args := []any{repoID}
232 if job != "" {
233 q += " AND job = ?"
234 args = append(args, job)
235 }
236 q += " ORDER BY number DESC LIMIT 1"
237 b, err := scanBuild(s.DB.QueryRow(q, args...))
238 if errors.Is(err, sql.ErrNoRows) {
239 return b, ErrNotFound
240 }
241 return b, err
242}
243
244// BuildsForCommit returns the newest build per job for one commit. A merge
245// request's checks are ci/<job> statuses; this is where their timing comes
246// from, in one query rather than one per check.
247func (s *Store) BuildsForCommit(repoID int64, sha string) (map[string]Build, error) {
248 rows, err := s.DB.Query(buildSelect+" WHERE repo_id = ? AND sha = ? ORDER BY number ASC", repoID, sha)
249 if err != nil {
250 return nil, err
251 }
252 defer rows.Close()
253 out := map[string]Build{}
254 for rows.Next() {
255 b, err := scanBuild(rows)
256 if err != nil {
257 return nil, err
258 }
259 out[b.Job] = b // ascending: the last row for a job wins
260 }
261 return out, rows.Err()
262}
263
264// Elapsed reports how long a build ran. Zero until it has both a start and
265// a finish, which is every state but success and failure.
266func (b Build) Elapsed() time.Duration {
267 const layout = "2006-01-02T15:04:05Z"
268 start, err := time.Parse(layout, b.StartedAt)
269 if err != nil {
270 return 0
271 }
272 end, err := time.Parse(layout, b.FinishedAt)
273 if err != nil {
274 return 0
275 }
276 if d := end.Sub(start); d > 0 {
277 return d.Round(time.Second)
278 }
279 return 0
280}