internal/store/builds.go
261 lines · 7799 bytes
1package store
2
3import (
4 "database/sql"
5 "errors"
6 "strings"
7 "time"
8)
9
10// Build is one CI job execution for one commit.
11type Build struct {
12 ID int64
13 RepoID int64
14 Number int64
15 Job string
16 SHA string
17 Ref string
18 Steps string // JSON array of shell commands
19 Status string // pending|running|success|failure
20 CreatedAt string
21 StartedAt string
22 FinishedAt string
23}
24
25// MaxBuildLog caps a build's stored log; appends past it are dropped.
26const MaxBuildLog = 2 << 20
27
28// CreateBuild allocates the per-repo build number in the same transaction
29// as the insert, like issue and MR numbers.
30func (s *Store) CreateBuild(repoID int64, job, sha, ref, stepsJSON string) (int64, error) {
31 tx, err := s.DB.Begin()
32 if err != nil {
33 return 0, err
34 }
35 defer tx.Rollback()
36 if _, err := tx.Exec("UPDATE repos SET build_counter = build_counter + 1 WHERE id = ?", repoID); err != nil {
37 return 0, err
38 }
39 var n int64
40 if err := tx.QueryRow("SELECT build_counter FROM repos WHERE id = ?", repoID).Scan(&n); err != nil {
41 return 0, err
42 }
43 if _, err := tx.Exec(
44 "INSERT INTO builds (repo_id, number, job, sha, ref, steps) VALUES (?, ?, ?, ?, ?, ?)",
45 repoID, n, job, sha, ref, stepsJSON); err != nil {
46 return 0, err
47 }
48 return n, tx.Commit()
49}
50
51const buildSelect = `
52 SELECT id, repo_id, number, job, sha, ref, steps, status, created_at, started_at, finished_at
53 FROM builds`
54
55func scanBuild(row interface{ Scan(...any) error }) (Build, error) {
56 var b Build
57 err := row.Scan(&b.ID, &b.RepoID, &b.Number, &b.Job, &b.SHA, &b.Ref, &b.Steps,
58 &b.Status, &b.CreatedAt, &b.StartedAt, &b.FinishedAt)
59 return b, err
60}
61
62// ClaimBuild atomically hands the oldest pending build to a runner.
63// ClaimBuild takes the oldest pending build and marks it running. A
64// non-empty repoIDs restricts the claim to those repositories, which is how
65// a runner on a machine that should not execute every repository's steps
66// limits what it picks up.
67func (s *Store) ClaimBuild(repoIDs []int64) (Build, bool, error) {
68 tx, err := s.DB.Begin()
69 if err != nil {
70 return Build{}, false, err
71 }
72 defer tx.Rollback()
73 query := "SELECT id FROM builds WHERE status = 'pending' ORDER BY id LIMIT 1"
74 args := []any{}
75 if len(repoIDs) > 0 {
76 marks := strings.TrimSuffix(strings.Repeat("?,", len(repoIDs)), ",")
77 query = "SELECT id FROM builds WHERE status = 'pending' AND repo_id IN (" +
78 marks + ") ORDER BY id LIMIT 1"
79 for _, id := range repoIDs {
80 args = append(args, id)
81 }
82 }
83 var id int64
84 err = tx.QueryRow(query, args...).Scan(&id)
85 if errors.Is(err, sql.ErrNoRows) {
86 return Build{}, false, nil
87 }
88 if err != nil {
89 return Build{}, false, err
90 }
91 if _, err := tx.Exec(
92 "UPDATE builds SET status = 'running', started_at = strftime('%Y-%m-%dT%H:%M:%SZ','now') WHERE id = ?", id); err != nil {
93 return Build{}, false, err
94 }
95 b, err := scanBuild(tx.QueryRow(buildSelect+" WHERE id = ?", id))
96 if err != nil {
97 return Build{}, false, err
98 }
99 return b, true, tx.Commit()
100}
101
102// StaleBuildDeadline is how long a claimed build may stay running before the
103// server gives up on it. Comfortably longer than the runner's own -timeout
104// (45m by default), so this only fires when the runner never reported at all —
105// it was killed, restarted, or lost the network mid-build.
106const StaleBuildDeadline = 90 * time.Minute
107
108// ReapStaleBuilds fails every build that has been running past the deadline and
109// returns them, so the caller can resolve their commit statuses. A runner that
110// dies between claiming a build and reporting it otherwise leaves the row
111// claimed forever, and the commit pending forever with it.
112func (s *Store) ReapStaleBuilds() ([]Build, error) {
113 cutoff := time.Now().UTC().Add(-StaleBuildDeadline).Format("2006-01-02T15:04:05Z")
114 rows, err := s.DB.Query(buildSelect+
115 " WHERE status = 'running' AND started_at != '' AND started_at < ?", cutoff)
116 if err != nil {
117 return nil, err
118 }
119 defer rows.Close()
120 var stale []Build
121 for rows.Next() {
122 b, err := scanBuild(rows)
123 if err != nil {
124 return nil, err
125 }
126 stale = append(stale, b)
127 }
128 if err := rows.Err(); err != nil {
129 return nil, err
130 }
131 for _, b := range stale {
132 if err := s.AppendBuildLog(b.ID, []byte(
133 "\nbuild abandoned: the runner never reported an outcome\n")); err != nil {
134 return nil, err
135 }
136 if err := s.FinishBuild(b.ID, "failure"); err != nil {
137 return nil, err
138 }
139 }
140 return stale, nil
141}
142
143// AppendBuildLog adds a chunk to the build's log, dropping bytes past the cap.
144func (s *Store) AppendBuildLog(id int64, chunk []byte) error {
145 _, err := s.DB.Exec(`
146 UPDATE builds SET log = log || ?
147 WHERE id = ? AND length(log) < ?`, chunk, id, MaxBuildLog)
148 return err
149}
150
151// FinishBuild records the outcome of a running build.
152func (s *Store) FinishBuild(id int64, status string) error {
153 res, err := s.DB.Exec(`
154 UPDATE builds SET status = ?, finished_at = strftime('%Y-%m-%dT%H:%M:%SZ','now')
155 WHERE id = ? AND status = 'running'`, status, id)
156 if err != nil {
157 return err
158 }
159 if n, _ := res.RowsAffected(); n == 0 {
160 return ErrNotFound
161 }
162 return nil
163}
164
165func (s *Store) BuildByID(id int64) (Build, error) {
166 b, err := scanBuild(s.DB.QueryRow(buildSelect+" WHERE id = ?", id))
167 if errors.Is(err, sql.ErrNoRows) {
168 return b, ErrNotFound
169 }
170 return b, err
171}
172
173func (s *Store) BuildByNumber(repoID, number int64) (Build, error) {
174 b, err := scanBuild(s.DB.QueryRow(buildSelect+" WHERE repo_id = ? AND number = ?", repoID, number))
175 if errors.Is(err, sql.ErrNoRows) {
176 return b, ErrNotFound
177 }
178 return b, err
179}
180
181func (s *Store) ListBuilds(repoID int64, limit int) ([]Build, error) {
182 rows, err := s.DB.Query(buildSelect+" WHERE repo_id = ? ORDER BY number DESC LIMIT ?", repoID, limit)
183 if err != nil {
184 return nil, err
185 }
186 defer rows.Close()
187 var out []Build
188 for rows.Next() {
189 b, err := scanBuild(rows)
190 if err != nil {
191 return nil, err
192 }
193 out = append(out, b)
194 }
195 return out, rows.Err()
196}
197
198// BuildLog returns the stored log bytes.
199func (s *Store) BuildLog(id int64) ([]byte, error) {
200 var log []byte
201 err := s.DB.QueryRow("SELECT log FROM builds WHERE id = ?", id).Scan(&log)
202 if errors.Is(err, sql.ErrNoRows) {
203 return nil, ErrNotFound
204 }
205 return log, err
206}
207
208// LatestBuild returns the newest build for a repo, optionally narrowed to
209// one job. It is what a status badge reports.
210func (s *Store) LatestBuild(repoID int64, job string) (Build, error) {
211 q := buildSelect + " WHERE repo_id = ?"
212 args := []any{repoID}
213 if job != "" {
214 q += " AND job = ?"
215 args = append(args, job)
216 }
217 q += " ORDER BY number DESC LIMIT 1"
218 b, err := scanBuild(s.DB.QueryRow(q, args...))
219 if errors.Is(err, sql.ErrNoRows) {
220 return b, ErrNotFound
221 }
222 return b, err
223}
224
225// BuildsForCommit returns the newest build per job for one commit. A merge
226// request's checks are ci/<job> statuses; this is where their timing comes
227// from, in one query rather than one per check.
228func (s *Store) BuildsForCommit(repoID int64, sha string) (map[string]Build, error) {
229 rows, err := s.DB.Query(buildSelect+" WHERE repo_id = ? AND sha = ? ORDER BY number ASC", repoID, sha)
230 if err != nil {
231 return nil, err
232 }
233 defer rows.Close()
234 out := map[string]Build{}
235 for rows.Next() {
236 b, err := scanBuild(rows)
237 if err != nil {
238 return nil, err
239 }
240 out[b.Job] = b // ascending: the last row for a job wins
241 }
242 return out, rows.Err()
243}
244
245// Elapsed reports how long a build ran. Zero until it has both a start and
246// a finish, which is every state but success and failure.
247func (b Build) Elapsed() time.Duration {
248 const layout = "2006-01-02T15:04:05Z"
249 start, err := time.Parse(layout, b.StartedAt)
250 if err != nil {
251 return 0
252 }
253 end, err := time.Parse(layout, b.FinishedAt)
254 if err != nil {
255 return 0
256 }
257 if d := end.Sub(start); d > 0 {
258 return d.Round(time.Second)
259 }
260 return 0
261}