e2e/reap_test.go
258 lines · 10665 bytes
1package e2e
2
3import (
4 "crypto/sha256"
5 "encoding/hex"
6 "encoding/json"
7 "fmt"
8 "os"
9 "path/filepath"
10 "strings"
11 "testing"
12 "time"
13)
14
15// A build claimed by a runner that never reports is failed by the
16// scheduler's tick, with no runner alive to trigger it.
17func TestStaleBuildReapedWithoutRunner(t *testing.T) {
18 t.Setenv("GITBAY_SCHED_TICK", "500ms")
19 t.Setenv("GITBAY_STALE_BUILD_DEADLINE", "2s")
20 inst := startInstance(t)
21 aliceKey := inst.newKey(t, "alice")
22 runnerKey := inst.newKey(t, "ci")
23 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
24 inst.admin(t, "admin", "user", "create", "ci", "--key", runnerKey+".pub", "--admin")
25 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
26 t.Fatal("repo create failed")
27 }
28 work := t.TempDir()
29 env := inst.gitEnv(aliceKey)
30 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
31 dir := filepath.Join(work, "w")
32 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
33 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte("jobs:\n ok:\n steps:\n - echo fine\n"), 0o644)
34 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
35 mustGit(t, dir, env, "add", ".")
36 mustGit(t, dir, env, "commit", "-q", "-m", "ci")
37 mustGit(t, dir, env, "push", "-q", "origin", "main")
38
39 // Claim the build the way a runner does, then vanish.
40 out, errOut, code := inst.ssh(t, runnerKey, "", "runner", "next", "--json")
41 if code != 0 || !strings.Contains(out, `"job":"ok"`) {
42 t.Fatalf("runner next: exit %d\n%s%s", code, out, errOut)
43 }
44 var claim struct {
45 Data struct {
46 SHA string `json:"sha"`
47 } `json:"data"`
48 }
49 json.Unmarshal([]byte(out), &claim)
50
51 status := func() (string, string) {
52 out, _, _ := inst.ssh(t, aliceKey, "", "build", "list", "alice/app", "--json")
53 var env struct {
54 Data []struct {
55 Status string `json:"status"`
56 } `json:"data"`
57 }
58 json.Unmarshal([]byte(out), &env)
59 st := ""
60 if len(env.Data) > 0 {
61 st = env.Data[0].Status
62 }
63 cs, _, _ := inst.ssh(t, aliceKey, "", "status", "list", "alice/app", claim.Data.SHA)
64 return st, cs
65 }
66 if st, _ := status(); st != "running" {
67 t.Fatalf("claimed build is %q, want running", st)
68 }
69 deadline := time.Now().Add(20 * time.Second)
70 for {
71 st, cs := status()
72 if st == "failure" && strings.Contains(cs, "build abandoned") {
73 break
74 }
75 if time.Now().After(deadline) {
76 t.Fatalf("never reaped: build %q, statuses:\n%s", st, cs)
77 }
78 time.Sleep(250 * time.Millisecond)
79 }
80 if out, _, _ := inst.ssh(t, aliceKey, "", "build", "log", "alice/app", "1"); !strings.Contains(out, "build abandoned") {
81 t.Fatalf("log lacks the abandonment note:\n%s", out)
82 }
83 // The reaper's work is counted, apart from builds runners reported.
84 if out, _, _ := inst.ssh(t, runnerKey, "", "admin", "runners", "--json"); !strings.Contains(out, `"reaped_24h":1`) ||
85 !strings.Contains(out, `"claimed_24h":1`) {
86 t.Fatalf("queue after reap:\n%s", out)
87 }
88}
89
90func TestAdminRunners(t *testing.T) {
91 t.Parallel()
92 inst := startInstance(t)
93 rootKey := inst.newKey(t, "root")
94 aliceKey := inst.newKey(t, "alice")
95 runnerKey := inst.newKey(t, "ci")
96 inst.admin(t, "admin", "user", "create", "root", "--key", rootKey+".pub", "--admin")
97 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
98 inst.admin(t, "admin", "user", "create", "ci", "--key", runnerKey+".pub", "--admin")
99 if _, _, code := inst.ssh(t, aliceKey, "", "admin", "runners"); code != 4 {
100 t.Fatal("non-admin listed runners")
101 }
102 if out, _, code := inst.ssh(t, rootKey, "", "admin", "runners", "--json"); code != 0 ||
103 strings.TrimSpace(out) != `{"protocol_version":1,"data":{"queue":{"pending":0,"claimed_24h":0,"claim_wait_avg_s":0,"claim_wait_max_s":0,"reaped_24h":0},"runners":[]}}` {
104 t.Fatalf("no runners yet: exit %d %s", code, out)
105 }
106 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
107 t.Fatal("repo create failed")
108 }
109 // An idle poll registers the runner with its scope.
110 if _, _, code := inst.ssh(t, runnerKey, "", "runner", "next", "alice/app"); code != 0 {
111 t.Fatal("runner next failed")
112 }
113 out, _, _ := inst.ssh(t, rootKey, "", "admin", "runners")
114 if !strings.HasPrefix(out, "queue: 0 pending; last 24h: 0 claimed") || !strings.Contains(out, "\nci\t") ||
115 !strings.Contains(out, "\talice/app\tidle") {
116 t.Fatalf("idle runner row:\n%s", out)
117 }
118 work := t.TempDir()
119 env := inst.gitEnv(aliceKey)
120 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
121 dir := filepath.Join(work, "w")
122 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
123 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte("jobs:\n ok:\n steps:\n - echo fine\n"), 0o644)
124 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
125 mustGit(t, dir, env, "add", ".")
126 mustGit(t, dir, env, "commit", "-q", "-m", "ci")
127 mustGit(t, dir, env, "push", "-q", "origin", "main")
128 out, _, code := inst.ssh(t, runnerKey, "", "runner", "next", "--json")
129 if code != 0 || !strings.Contains(out, `"job":"ok"`) {
130 t.Fatalf("claim: %s", out)
131 }
132 var claim struct {
133 Data struct {
134 ID int64 `json:"id"`
135 } `json:"data"`
136 }
137 json.Unmarshal([]byte(out), &claim)
138 if out, _, _ := inst.ssh(t, rootKey, "", "admin", "runners"); !strings.Contains(out, "\tany\talice/app #1 ok since ") {
139 t.Fatalf("holding runner row:\n%s", out)
140 }
141 if _, _, code := inst.ssh(t, runnerKey, "", "runner", "done", fmt.Sprint(claim.Data.ID), "success"); code != 0 {
142 t.Fatal("runner done failed")
143 }
144 if out, _, _ := inst.ssh(t, rootKey, "", "admin", "runners"); !strings.Contains(out, "\tany\tidle") ||
145 !strings.HasPrefix(out, "queue: 0 pending; last 24h: 1 claimed, wait avg ") {
146 t.Fatalf("runner still holds a build after done:\n%s", out)
147 }
148 // Host-local, the same read.
149 if out := inst.admin(t, "admin", "runners", "--json"); !strings.Contains(out, `"username":"ci"`) {
150 t.Fatalf("host runners:\n%s", out)
151 }
152}
153
154// /healthz is unauthenticated, cache-free, and says which build serves.
155func TestHealthz(t *testing.T) {
156 t.Parallel()
157 inst := startInstance(t)
158 status, body := inst.get(t, "/healthz")
159 if status != 200 || !strings.Contains(body, `"ok":true`) || !strings.Contains(body, `"commit":"`) {
160 t.Fatalf("healthz: %d %s", status, body)
161 }
162 // The name is reserved: no account can shadow the route.
163 if _, errOut, code := inst.ssh(t, inst.newKey(t, "x"), "", "register", "--username", "healthz"); code == 0 {
164 t.Fatalf("healthz registered as a username: %s", errOut)
165 }
166}
167
168// gc --lfs removes objects no pointer names, keeps referenced ones, and
169// leaves anything young enough to be an upload ahead of its push.
170func TestGCLFSOrphans(t *testing.T) {
171 t.Parallel()
172 inst := startInstance(t)
173 aliceKey := inst.newKey(t, "alice")
174 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
175 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/big"); code != 0 {
176 t.Fatal("repo create failed")
177 }
178 put := func(content string, age time.Duration) string {
179 t.Helper()
180 sum := sha256.Sum256([]byte(content))
181 oid := hex.EncodeToString(sum[:])
182 path := filepath.Join(inst.root, "lfs", oid[:2], oid[2:4], oid)
183 os.MkdirAll(filepath.Dir(path), 0o755)
184 os.WriteFile(path, []byte(content), 0o644)
185 os.Chtimes(path, time.Now().Add(-age), time.Now().Add(-age))
186 return oid
187 }
188 kept := put("referenced payload", 48*time.Hour)
189 orphan := put("nobody points here", 48*time.Hour)
190 young := put("just uploaded", time.Minute)
191
192 work := t.TempDir()
193 env := inst.gitEnv(aliceKey)
194 mustGit(t, work, env, "clone", inst.sshURL("alice/big"), "w")
195 dir := filepath.Join(work, "w")
196 pointer := fmt.Sprintf("version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %d\n", kept, len("referenced payload"))
197 os.WriteFile(filepath.Join(dir, "data.bin"), []byte(pointer), 0o644)
198 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
199 mustGit(t, dir, env, "add", ".")
200 mustGit(t, dir, env, "commit", "-q", "-m", "pointer")
201 mustGit(t, dir, env, "push", "-q", "origin", "main")
202
203 if out := inst.admin(t, "admin", "stats", "--json"); !strings.Contains(out, `"lfs_bytes":`) || strings.Contains(out, `"lfs_bytes":0`) {
204 t.Fatalf("stats lfs bytes:\n%s", out)
205 }
206 out := inst.admin(t, "admin", "gc", "--lfs")
207 if !strings.Contains(out, "lfs\t1 referenced, removed 1 orphans") {
208 t.Fatalf("gc --lfs:\n%s", out)
209 }
210 exists := func(oid string) bool {
211 _, err := os.Stat(filepath.Join(inst.root, "lfs", oid[:2], oid[2:4], oid))
212 return err == nil
213 }
214 if !exists(kept) || exists(orphan) || !exists(young) {
215 t.Fatalf("after gc: kept=%v orphan=%v young=%v", exists(kept), exists(orphan), exists(young))
216 }
217}
218
219// backup --verify reads an archive back and says whether a restore would
220// have what the database expects.
221func TestBackupVerify(t *testing.T) {
222 t.Parallel()
223 inst := startInstance(t)
224 aliceKey := inst.newKey(t, "alice")
225 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
226 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/keep"); code != 0 {
227 t.Fatal("repo create failed")
228 }
229 full := filepath.Join(t.TempDir(), "full.tar.gz")
230 inst.admin(t, "admin", "backup", "--out", full)
231 if out := inst.admin(t, "admin", "backup", "--verify", full); !strings.Contains(out, "integrity ok, 1 repositories in the database, 1 in the archive") {
232 t.Fatalf("verify full:\n%s", out)
233 }
234 dbOnly := filepath.Join(t.TempDir(), "db.tar.gz")
235 inst.admin(t, "admin", "backup", "--db-only", "--out", dbOnly)
236 if out := inst.admin(t, "admin", "backup", "--verify", dbOnly); !strings.Contains(out, "database only; integrity ok, 1 repositories in the database") {
237 t.Fatalf("verify db-only:\n%s", out)
238 }
239 // A truncated archive is named as damaged, not reported healthy.
240 raw, _ := os.ReadFile(full)
241 cut := filepath.Join(t.TempDir(), "cut.tar.gz")
242 os.WriteFile(cut, raw[:len(raw)/2], 0o644)
243 if out := inst.forgedAdminErr(t, "admin", "backup", "--verify", cut); !strings.Contains(out, "damaged") && !strings.Contains(out, "unexpected EOF") {
244 t.Fatalf("verify truncated:\n%s", out)
245 }
246 // A repository the database names but the archive lacks fails it.
247 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/late"); code != 0 {
248 t.Fatal("repo create failed")
249 }
250 late := filepath.Join(t.TempDir(), "late.tar.gz")
251 inst.admin(t, "admin", "backup", "--out", late)
252 os.RemoveAll(filepath.Join(inst.root, "repos", "alice", "late.git"))
253 stale := filepath.Join(t.TempDir(), "stale.tar.gz")
254 inst.admin(t, "admin", "backup", "--out", stale)
255 if out := inst.forgedAdminErr(t, "admin", "backup", "--verify", stale); !strings.Contains(out, "not in the archive: alice/late") {
256 t.Fatalf("verify missing repo:\n%s", out)
257 }
258}