e2e/reap_test.go
254 lines · 10609 bytes
1package e2e
2
3import (
4 "crypto/sha256"
5 "encoding/hex"
6 "encoding/json"
7 "fmt"
8 "os"
9 "path/filepath"
10 "strings"
11 "testing"
12 "time"
13)
14
15// A build claimed by a runner that never reports is failed by the
16// scheduler's tick, with no runner alive to trigger it.
17func TestStaleBuildReapedWithoutRunner(t *testing.T) {
18 t.Setenv("GITBAY_SCHED_TICK", "500ms")
19 t.Setenv("GITBAY_STALE_BUILD_DEADLINE", "2s")
20 inst := startInstance(t)
21 aliceKey := inst.newKey(t, "alice")
22 runnerKey := inst.newKey(t, "ci")
23 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
24 inst.admin(t, "admin", "user", "create", "ci", "--key", runnerKey+".pub", "--admin")
25 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
26 t.Fatal("repo create failed")
27 }
28 work := t.TempDir()
29 env := inst.gitEnv(aliceKey)
30 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
31 dir := filepath.Join(work, "w")
32 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
33 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte("jobs:\n ok:\n steps:\n - echo fine\n"), 0o644)
34 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
35 mustGit(t, dir, env, "add", ".")
36 mustGit(t, dir, env, "commit", "-q", "-m", "ci")
37 mustGit(t, dir, env, "push", "-q", "origin", "main")
38
39 // Claim the build the way a runner does, then vanish.
40 out, errOut, code := inst.ssh(t, runnerKey, "", "runner", "next", "--json")
41 if code != 0 || !strings.Contains(out, `"job":"ok"`) {
42 t.Fatalf("runner next: exit %d\n%s%s", code, out, errOut)
43 }
44 var claim struct {
45 Data struct {
46 SHA string `json:"sha"`
47 } `json:"data"`
48 }
49 json.Unmarshal([]byte(out), &claim)
50
51 status := func() (string, string) {
52 out, _, _ := inst.ssh(t, aliceKey, "", "build", "list", "alice/app", "--json")
53 var env struct {
54 Data []struct {
55 Status string `json:"status"`
56 } `json:"data"`
57 }
58 json.Unmarshal([]byte(out), &env)
59 st := ""
60 if len(env.Data) > 0 {
61 st = env.Data[0].Status
62 }
63 cs, _, _ := inst.ssh(t, aliceKey, "", "status", "list", "alice/app", claim.Data.SHA)
64 return st, cs
65 }
66 if st, _ := status(); st != "running" {
67 t.Fatalf("claimed build is %q, want running", st)
68 }
69 deadline := time.Now().Add(20 * time.Second)
70 for {
71 st, cs := status()
72 if st == "failure" && strings.Contains(cs, "build abandoned") {
73 break
74 }
75 if time.Now().After(deadline) {
76 t.Fatalf("never reaped: build %q, statuses:\n%s", st, cs)
77 }
78 time.Sleep(250 * time.Millisecond)
79 }
80 if out, _, _ := inst.ssh(t, aliceKey, "", "build", "log", "alice/app", "1"); !strings.Contains(out, "build abandoned") {
81 t.Fatalf("log lacks the abandonment note:\n%s", out)
82 }
83 // The reaper's work is counted, apart from builds runners reported.
84 if out, _, _ := inst.ssh(t, runnerKey, "", "admin", "runners", "--json"); !strings.Contains(out, `"reaped_24h":1`) ||
85 !strings.Contains(out, `"claimed_24h":1`) {
86 t.Fatalf("queue after reap:\n%s", out)
87 }
88}
89
90func TestAdminRunners(t *testing.T) {
91 inst := startInstance(t)
92 rootKey := inst.newKey(t, "root")
93 aliceKey := inst.newKey(t, "alice")
94 runnerKey := inst.newKey(t, "ci")
95 inst.admin(t, "admin", "user", "create", "root", "--key", rootKey+".pub", "--admin")
96 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
97 inst.admin(t, "admin", "user", "create", "ci", "--key", runnerKey+".pub", "--admin")
98 if _, _, code := inst.ssh(t, aliceKey, "", "admin", "runners"); code != 4 {
99 t.Fatal("non-admin listed runners")
100 }
101 if out, _, code := inst.ssh(t, rootKey, "", "admin", "runners", "--json"); code != 0 ||
102 strings.TrimSpace(out) != `{"protocol_version":1,"data":{"queue":{"pending":0,"claimed_24h":0,"claim_wait_avg_s":0,"claim_wait_max_s":0,"reaped_24h":0},"runners":[]}}` {
103 t.Fatalf("no runners yet: exit %d %s", code, out)
104 }
105 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
106 t.Fatal("repo create failed")
107 }
108 // An idle poll registers the runner with its scope.
109 if _, _, code := inst.ssh(t, runnerKey, "", "runner", "next", "alice/app"); code != 0 {
110 t.Fatal("runner next failed")
111 }
112 out, _, _ := inst.ssh(t, rootKey, "", "admin", "runners")
113 if !strings.HasPrefix(out, "queue: 0 pending; last 24h: 0 claimed") || !strings.Contains(out, "\nci\t") ||
114 !strings.Contains(out, "\talice/app\tidle") {
115 t.Fatalf("idle runner row:\n%s", out)
116 }
117 work := t.TempDir()
118 env := inst.gitEnv(aliceKey)
119 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
120 dir := filepath.Join(work, "w")
121 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
122 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte("jobs:\n ok:\n steps:\n - echo fine\n"), 0o644)
123 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
124 mustGit(t, dir, env, "add", ".")
125 mustGit(t, dir, env, "commit", "-q", "-m", "ci")
126 mustGit(t, dir, env, "push", "-q", "origin", "main")
127 out, _, code := inst.ssh(t, runnerKey, "", "runner", "next", "--json")
128 if code != 0 || !strings.Contains(out, `"job":"ok"`) {
129 t.Fatalf("claim: %s", out)
130 }
131 var claim struct {
132 Data struct {
133 ID int64 `json:"id"`
134 } `json:"data"`
135 }
136 json.Unmarshal([]byte(out), &claim)
137 if out, _, _ := inst.ssh(t, rootKey, "", "admin", "runners"); !strings.Contains(out, "\tany\talice/app #1 ok since ") {
138 t.Fatalf("holding runner row:\n%s", out)
139 }
140 if _, _, code := inst.ssh(t, runnerKey, "", "runner", "done", fmt.Sprint(claim.Data.ID), "success"); code != 0 {
141 t.Fatal("runner done failed")
142 }
143 if out, _, _ := inst.ssh(t, rootKey, "", "admin", "runners"); !strings.Contains(out, "\tany\tidle") ||
144 !strings.HasPrefix(out, "queue: 0 pending; last 24h: 1 claimed, wait avg ") {
145 t.Fatalf("runner still holds a build after done:\n%s", out)
146 }
147 // Host-local, the same read.
148 if out := inst.admin(t, "admin", "runners", "--json"); !strings.Contains(out, `"username":"ci"`) {
149 t.Fatalf("host runners:\n%s", out)
150 }
151}
152
153// /healthz is unauthenticated, cache-free, and says which build serves.
154func TestHealthz(t *testing.T) {
155 inst := startInstance(t)
156 status, body := inst.get(t, "/healthz")
157 if status != 200 || !strings.Contains(body, `"ok":true`) || !strings.Contains(body, `"commit":"`) {
158 t.Fatalf("healthz: %d %s", status, body)
159 }
160 // The name is reserved: no account can shadow the route.
161 if _, errOut, code := inst.ssh(t, inst.newKey(t, "x"), "", "register", "--username", "healthz"); code == 0 {
162 t.Fatalf("healthz registered as a username: %s", errOut)
163 }
164}
165
166// gc --lfs removes objects no pointer names, keeps referenced ones, and
167// leaves anything young enough to be an upload ahead of its push.
168func TestGCLFSOrphans(t *testing.T) {
169 inst := startInstance(t)
170 aliceKey := inst.newKey(t, "alice")
171 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
172 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/big"); code != 0 {
173 t.Fatal("repo create failed")
174 }
175 put := func(content string, age time.Duration) string {
176 t.Helper()
177 sum := sha256.Sum256([]byte(content))
178 oid := hex.EncodeToString(sum[:])
179 path := filepath.Join(inst.root, "lfs", oid[:2], oid[2:4], oid)
180 os.MkdirAll(filepath.Dir(path), 0o755)
181 os.WriteFile(path, []byte(content), 0o644)
182 os.Chtimes(path, time.Now().Add(-age), time.Now().Add(-age))
183 return oid
184 }
185 kept := put("referenced payload", 48*time.Hour)
186 orphan := put("nobody points here", 48*time.Hour)
187 young := put("just uploaded", time.Minute)
188
189 work := t.TempDir()
190 env := inst.gitEnv(aliceKey)
191 mustGit(t, work, env, "clone", inst.sshURL("alice/big"), "w")
192 dir := filepath.Join(work, "w")
193 pointer := fmt.Sprintf("version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %d\n", kept, len("referenced payload"))
194 os.WriteFile(filepath.Join(dir, "data.bin"), []byte(pointer), 0o644)
195 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
196 mustGit(t, dir, env, "add", ".")
197 mustGit(t, dir, env, "commit", "-q", "-m", "pointer")
198 mustGit(t, dir, env, "push", "-q", "origin", "main")
199
200 if out := inst.admin(t, "admin", "stats", "--json"); !strings.Contains(out, `"lfs_bytes":`) || strings.Contains(out, `"lfs_bytes":0`) {
201 t.Fatalf("stats lfs bytes:\n%s", out)
202 }
203 out := inst.admin(t, "admin", "gc", "--lfs")
204 if !strings.Contains(out, "lfs\t1 referenced, removed 1 orphans") {
205 t.Fatalf("gc --lfs:\n%s", out)
206 }
207 exists := func(oid string) bool {
208 _, err := os.Stat(filepath.Join(inst.root, "lfs", oid[:2], oid[2:4], oid))
209 return err == nil
210 }
211 if !exists(kept) || exists(orphan) || !exists(young) {
212 t.Fatalf("after gc: kept=%v orphan=%v young=%v", exists(kept), exists(orphan), exists(young))
213 }
214}
215
216// backup --verify reads an archive back and says whether a restore would
217// have what the database expects.
218func TestBackupVerify(t *testing.T) {
219 inst := startInstance(t)
220 aliceKey := inst.newKey(t, "alice")
221 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
222 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/keep"); code != 0 {
223 t.Fatal("repo create failed")
224 }
225 full := filepath.Join(t.TempDir(), "full.tar.gz")
226 inst.admin(t, "admin", "backup", "--out", full)
227 if out := inst.admin(t, "admin", "backup", "--verify", full); !strings.Contains(out, "integrity ok, 1 repositories in the database, 1 in the archive") {
228 t.Fatalf("verify full:\n%s", out)
229 }
230 dbOnly := filepath.Join(t.TempDir(), "db.tar.gz")
231 inst.admin(t, "admin", "backup", "--db-only", "--out", dbOnly)
232 if out := inst.admin(t, "admin", "backup", "--verify", dbOnly); !strings.Contains(out, "database only; integrity ok, 1 repositories in the database") {
233 t.Fatalf("verify db-only:\n%s", out)
234 }
235 // A truncated archive is named as damaged, not reported healthy.
236 raw, _ := os.ReadFile(full)
237 cut := filepath.Join(t.TempDir(), "cut.tar.gz")
238 os.WriteFile(cut, raw[:len(raw)/2], 0o644)
239 if out := inst.forgedAdminErr(t, "admin", "backup", "--verify", cut); !strings.Contains(out, "damaged") && !strings.Contains(out, "unexpected EOF") {
240 t.Fatalf("verify truncated:\n%s", out)
241 }
242 // A repository the database names but the archive lacks fails it.
243 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/late"); code != 0 {
244 t.Fatal("repo create failed")
245 }
246 late := filepath.Join(t.TempDir(), "late.tar.gz")
247 inst.admin(t, "admin", "backup", "--out", late)
248 os.RemoveAll(filepath.Join(inst.root, "repos", "alice", "late.git"))
249 stale := filepath.Join(t.TempDir(), "stale.tar.gz")
250 inst.admin(t, "admin", "backup", "--out", stale)
251 if out := inst.forgedAdminErr(t, "admin", "backup", "--verify", stale); !strings.Contains(out, "not in the archive: alice/late") {
252 t.Fatalf("verify missing repo:\n%s", out)
253 }
254}