e2e/reap_test.go
259 lines · 10756 bytes
1package e2e
2
3import (
4 "crypto/sha256"
5 "encoding/hex"
6 "encoding/json"
7 "fmt"
8 "os"
9 "path/filepath"
10 "regexp"
11 "strings"
12 "testing"
13 "time"
14)
15
16// A build claimed by a runner that never reports is failed by the
17// scheduler's tick, with no runner alive to trigger it.
18func TestStaleBuildReapedWithoutRunner(t *testing.T) {
19 t.Setenv("GITBAY_SCHED_TICK", "500ms")
20 t.Setenv("GITBAY_STALE_BUILD_DEADLINE", "2s")
21 inst := startInstance(t)
22 aliceKey := inst.newKey(t, "alice")
23 runnerKey := inst.newKey(t, "ci")
24 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
25 inst.admin(t, "admin", "user", "create", "ci", "--key", runnerKey+".pub", "--admin")
26 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
27 t.Fatal("repo create failed")
28 }
29 work := t.TempDir()
30 env := inst.gitEnv(aliceKey)
31 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
32 dir := filepath.Join(work, "w")
33 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
34 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte("jobs:\n ok:\n steps:\n - echo fine\n"), 0o644)
35 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
36 mustGit(t, dir, env, "add", ".")
37 mustGit(t, dir, env, "commit", "-q", "-m", "ci")
38 mustGit(t, dir, env, "push", "-q", "origin", "main")
39
40 // Claim the build the way a runner does, then vanish.
41 out, errOut, code := inst.ssh(t, runnerKey, "", "runner", "next", "--json")
42 if code != 0 || !strings.Contains(out, `"job":"ok"`) {
43 t.Fatalf("runner next: exit %d\n%s%s", code, out, errOut)
44 }
45 var claim struct {
46 Data struct {
47 SHA string `json:"sha"`
48 } `json:"data"`
49 }
50 json.Unmarshal([]byte(out), &claim)
51
52 status := func() (string, string) {
53 out, _, _ := inst.ssh(t, aliceKey, "", "build", "list", "alice/app", "--json")
54 var env struct {
55 Data []struct {
56 Status string `json:"status"`
57 } `json:"data"`
58 }
59 json.Unmarshal([]byte(out), &env)
60 st := ""
61 if len(env.Data) > 0 {
62 st = env.Data[0].Status
63 }
64 cs, _, _ := inst.ssh(t, aliceKey, "", "status", "list", "alice/app", claim.Data.SHA)
65 return st, cs
66 }
67 if st, _ := status(); st != "running" {
68 t.Fatalf("claimed build is %q, want running", st)
69 }
70 deadline := time.Now().Add(20 * time.Second)
71 for {
72 st, cs := status()
73 if st == "failure" && strings.Contains(cs, "build abandoned") {
74 break
75 }
76 if time.Now().After(deadline) {
77 t.Fatalf("never reaped: build %q, statuses:\n%s", st, cs)
78 }
79 time.Sleep(250 * time.Millisecond)
80 }
81 if out, _, _ := inst.ssh(t, aliceKey, "", "build", "log", "alice/app", "1"); !strings.Contains(out, "build abandoned") {
82 t.Fatalf("log lacks the abandonment note:\n%s", out)
83 }
84 // The reaper's work is counted, apart from builds runners reported.
85 if out, _, _ := inst.ssh(t, runnerKey, "", "admin", "runners", "--json"); !strings.Contains(out, `"reaped_24h":1`) ||
86 !strings.Contains(out, `"claimed_24h":1`) {
87 t.Fatalf("queue after reap:\n%s", out)
88 }
89}
90
91func TestAdminRunners(t *testing.T) {
92 t.Parallel()
93 inst := startInstance(t)
94 rootKey := inst.newKey(t, "root")
95 aliceKey := inst.newKey(t, "alice")
96 runnerKey := inst.newKey(t, "ci")
97 inst.admin(t, "admin", "user", "create", "root", "--key", rootKey+".pub", "--admin")
98 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
99 inst.admin(t, "admin", "user", "create", "ci", "--key", runnerKey+".pub", "--admin")
100 if _, _, code := inst.ssh(t, aliceKey, "", "admin", "runners"); code != 4 {
101 t.Fatal("non-admin listed runners")
102 }
103 if out, _, code := inst.ssh(t, rootKey, "", "admin", "runners", "--json"); code != 0 ||
104 strings.TrimSpace(out) != `{"protocol_version":1,"data":{"queue":{"pending":0,"claimed_24h":0,"claim_wait_avg_s":0,"claim_wait_max_s":0,"reaped_24h":0},"runners":[]}}` {
105 t.Fatalf("no runners yet: exit %d %s", code, out)
106 }
107 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
108 t.Fatal("repo create failed")
109 }
110 // An idle poll registers the runner with its scope.
111 if _, _, code := inst.ssh(t, runnerKey, "", "runner", "next", "alice/app"); code != 0 {
112 t.Fatal("runner next failed")
113 }
114 out, _, _ := inst.ssh(t, rootKey, "", "admin", "runners")
115 if !regexp.MustCompile(`pending\s+0`).MatchString(out) || !regexp.MustCompile(`claimed 24h\s+0`).MatchString(out) || !strings.Contains(out, "\nci\t") ||
116 !strings.Contains(out, "\talice/app\tidle") {
117 t.Fatalf("idle runner row:\n%s", out)
118 }
119 work := t.TempDir()
120 env := inst.gitEnv(aliceKey)
121 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
122 dir := filepath.Join(work, "w")
123 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
124 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte("jobs:\n ok:\n steps:\n - echo fine\n"), 0o644)
125 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
126 mustGit(t, dir, env, "add", ".")
127 mustGit(t, dir, env, "commit", "-q", "-m", "ci")
128 mustGit(t, dir, env, "push", "-q", "origin", "main")
129 out, _, code := inst.ssh(t, runnerKey, "", "runner", "next", "--json")
130 if code != 0 || !strings.Contains(out, `"job":"ok"`) {
131 t.Fatalf("claim: %s", out)
132 }
133 var claim struct {
134 Data struct {
135 ID int64 `json:"id"`
136 } `json:"data"`
137 }
138 json.Unmarshal([]byte(out), &claim)
139 if out, _, _ := inst.ssh(t, rootKey, "", "admin", "runners"); !strings.Contains(out, "\tany\talice/app #1 ok since ") {
140 t.Fatalf("holding runner row:\n%s", out)
141 }
142 if _, _, code := inst.ssh(t, runnerKey, "", "runner", "done", fmt.Sprint(claim.Data.ID), "success"); code != 0 {
143 t.Fatal("runner done failed")
144 }
145 if out, _, _ := inst.ssh(t, rootKey, "", "admin", "runners"); !strings.Contains(out, "\tany\tidle") ||
146 !regexp.MustCompile(`pending\s+0`).MatchString(out) || !regexp.MustCompile(`claimed 24h\s+1`).MatchString(out) {
147 t.Fatalf("runner still holds a build after done:\n%s", out)
148 }
149 // Host-local, the same read.
150 if out := inst.admin(t, "admin", "runners", "--json"); !strings.Contains(out, `"username":"ci"`) {
151 t.Fatalf("host runners:\n%s", out)
152 }
153}
154
155// /healthz is unauthenticated, cache-free, and says which build serves.
156func TestHealthz(t *testing.T) {
157 t.Parallel()
158 inst := startInstance(t)
159 status, body := inst.get(t, "/healthz")
160 if status != 200 || !strings.Contains(body, `"ok":true`) || !strings.Contains(body, `"commit":"`) {
161 t.Fatalf("healthz: %d %s", status, body)
162 }
163 // The name is reserved: no account can shadow the route.
164 if _, errOut, code := inst.ssh(t, inst.newKey(t, "x"), "", "register", "--username", "healthz"); code == 0 {
165 t.Fatalf("healthz registered as a username: %s", errOut)
166 }
167}
168
169// gc --lfs removes objects no pointer names, keeps referenced ones, and
170// leaves anything young enough to be an upload ahead of its push.
171func TestGCLFSOrphans(t *testing.T) {
172 t.Parallel()
173 inst := startInstance(t)
174 aliceKey := inst.newKey(t, "alice")
175 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
176 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/big"); code != 0 {
177 t.Fatal("repo create failed")
178 }
179 put := func(content string, age time.Duration) string {
180 t.Helper()
181 sum := sha256.Sum256([]byte(content))
182 oid := hex.EncodeToString(sum[:])
183 path := filepath.Join(inst.root, "lfs", oid[:2], oid[2:4], oid)
184 os.MkdirAll(filepath.Dir(path), 0o755)
185 os.WriteFile(path, []byte(content), 0o644)
186 os.Chtimes(path, time.Now().Add(-age), time.Now().Add(-age))
187 return oid
188 }
189 kept := put("referenced payload", 48*time.Hour)
190 orphan := put("nobody points here", 48*time.Hour)
191 young := put("just uploaded", time.Minute)
192
193 work := t.TempDir()
194 env := inst.gitEnv(aliceKey)
195 mustGit(t, work, env, "clone", inst.sshURL("alice/big"), "w")
196 dir := filepath.Join(work, "w")
197 pointer := fmt.Sprintf("version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %d\n", kept, len("referenced payload"))
198 os.WriteFile(filepath.Join(dir, "data.bin"), []byte(pointer), 0o644)
199 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
200 mustGit(t, dir, env, "add", ".")
201 mustGit(t, dir, env, "commit", "-q", "-m", "pointer")
202 mustGit(t, dir, env, "push", "-q", "origin", "main")
203
204 if out := inst.admin(t, "admin", "stats", "--json"); !strings.Contains(out, `"lfs_bytes":`) || strings.Contains(out, `"lfs_bytes":0`) {
205 t.Fatalf("stats lfs bytes:\n%s", out)
206 }
207 out := inst.admin(t, "admin", "gc", "--lfs")
208 if !strings.Contains(out, "lfs\t1 referenced, removed 1 orphans") {
209 t.Fatalf("gc --lfs:\n%s", out)
210 }
211 exists := func(oid string) bool {
212 _, err := os.Stat(filepath.Join(inst.root, "lfs", oid[:2], oid[2:4], oid))
213 return err == nil
214 }
215 if !exists(kept) || exists(orphan) || !exists(young) {
216 t.Fatalf("after gc: kept=%v orphan=%v young=%v", exists(kept), exists(orphan), exists(young))
217 }
218}
219
220// backup --verify reads an archive back and says whether a restore would
221// have what the database expects.
222func TestBackupVerify(t *testing.T) {
223 t.Parallel()
224 inst := startInstance(t)
225 aliceKey := inst.newKey(t, "alice")
226 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
227 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/keep"); code != 0 {
228 t.Fatal("repo create failed")
229 }
230 full := filepath.Join(t.TempDir(), "full.tar.gz")
231 inst.admin(t, "admin", "backup", "--out", full)
232 if out := inst.admin(t, "admin", "backup", "--verify", full); !strings.Contains(out, "integrity ok, 1 repositories in the database, 1 in the archive") {
233 t.Fatalf("verify full:\n%s", out)
234 }
235 dbOnly := filepath.Join(t.TempDir(), "db.tar.gz")
236 inst.admin(t, "admin", "backup", "--db-only", "--out", dbOnly)
237 if out := inst.admin(t, "admin", "backup", "--verify", dbOnly); !strings.Contains(out, "database only; integrity ok, 1 repositories in the database") {
238 t.Fatalf("verify db-only:\n%s", out)
239 }
240 // A truncated archive is named as damaged, not reported healthy.
241 raw, _ := os.ReadFile(full)
242 cut := filepath.Join(t.TempDir(), "cut.tar.gz")
243 os.WriteFile(cut, raw[:len(raw)/2], 0o644)
244 if out := inst.forgedAdminErr(t, "admin", "backup", "--verify", cut); !strings.Contains(out, "damaged") && !strings.Contains(out, "unexpected EOF") {
245 t.Fatalf("verify truncated:\n%s", out)
246 }
247 // A repository the database names but the archive lacks fails it.
248 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/late"); code != 0 {
249 t.Fatal("repo create failed")
250 }
251 late := filepath.Join(t.TempDir(), "late.tar.gz")
252 inst.admin(t, "admin", "backup", "--out", late)
253 os.RemoveAll(filepath.Join(inst.root, "repos", "alice", "late.git"))
254 stale := filepath.Join(t.TempDir(), "stale.tar.gz")
255 inst.admin(t, "admin", "backup", "--out", stale)
256 if out := inst.forgedAdminErr(t, "admin", "backup", "--verify", stale); !strings.Contains(out, "not in the archive: alice/late") {
257 t.Fatalf("verify missing repo:\n%s", out)
258 }
259}